from itertools import product
import numpy as np
from numpy.testing import assert_almost_equal, assert_array_almost_equal

from sklearn import datasets
from sklearn import manifold
from sklearn import neighbors
from sklearn import pipeline
from sklearn import preprocessing
from sklearn.utils.testing import assert_less

eigen_solvers = ['auto', 'dense', 'arpack']
path_methods = ['auto', 'FW', 'D']

def test_isomap_simple_grid():
    # Isomap should preserve distances when all neighbors are used
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # distances from each point to all others
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()
            assert_array_almost_equal(G, G_iso)

def test_isomap_reconstruction_error():
    # Same setup as in test_isomap_simple_grid, with an added dimension
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # add noise in a third dimension
    rng = np.random.RandomState(0)
    noise = 0.1 * rng.randn(Npts, 1)
    X = np.concatenate((X, noise), 1)

    # compute input kernel
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    centerer = preprocessing.KernelCenterer()
    K = centerer.fit_transform(-0.5 * G ** 2)

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            # compute output kernel
            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()

            K_iso = centerer.fit_transform(-0.5 * G_iso ** 2)

            # make sure error agrees
            reconstruction_error = np.linalg.norm(K - K_iso) / Npts
            assert_almost_equal(reconstruction_error,
                                clf.reconstruction_error())

def test_transform():
    n_samples = 200
    n_components = 10
    noise_scale = 0.01

    # Create S-curve dataset
    X, y = datasets.samples_generator.make_s_curve(n_samples, random_state=0)

    # Compute isomap embedding
    iso = manifold.Isomap(n_components, 2)
    X_iso = iso.fit_transform(X)

    # Re-embed a noisy version of the points
    rng = np.random.RandomState(0)
    noise = noise_scale * rng.randn(*X.shape)
    X_iso2 = iso.transform(X + noise)

    # Make sure the rms error on re-embedding is comparable to noise_scale
    assert_less(np.sqrt(np.mean((X_iso - X_iso2) ** 2)), 2 * noise_scale)

def test_pipeline():
    # check that Isomap works fine as a transformer in a Pipeline
    # only checks that no error is raised.
    # TODO check that it actually does something useful
    X, y = datasets.make_blobs(random_state=0)
    clf = pipeline.Pipeline(
        [('isomap', manifold.Isomap()),
         ('clf', neighbors.KNeighborsClassifier())])
    clf.fit(X, y)
    assert_less(.9, clf.score(X, y))


# -*- coding: utf-8 -*-
#
# Hooks module for py2exe.
# Inspired by cx_freeze's hooks.py, which is:
#
#    Copyright © 2007-2013, Anthony Tuininga.
#    Copyright © 2001-2006, Computronix (Canada) Ltd., Edmonton, Alberta, Canada.
#    All rights reserved.
#
import os, sys

# Exclude modules that the standard library imports (conditionally),
# but which are not present on windows.
#
# _memimporter can be excluded because it is built into the run-stub.
windows_excludes = """
_curses
_dummy_threading
_emx_link
_gestalt
_posixsubprocess
ce
clr
console
fcntl
grp
java
org
os2
posix
pwd
site
termios
vms_lib
_memimporter
""".split()

def init_finder(finder):
    # what about renamed functions, like urllib.pathname2url?
    #
    # We should use ignore() for Python 2 names so that my py2to3
    # importhook works.  For modules that are not present on Windows,
    # we should probably use excludes.append()
    finder.excludes.extend(windows_excludes)

    # python2 modules are ignored (but not excluded)
    finder.ignore("BaseHTTPServer")
    finder.ignore("ConfigParser")
    finder.ignore("IronPython")
    finder.ignore("SimpleHTTPServer")
    finder.ignore("StringIO")
    finder.ignore("__builtin__")
    finder.ignore("_winreg")
    finder.ignore("cPickle")
    finder.ignore("cStringIO")
    finder.ignore("commands")
    finder.ignore("compiler")
    finder.ignore("copy_reg")
    finder.ignore("dummy_thread")
    finder.ignore("future_builtins")
    finder.ignore("htmlentitydefs")
    finder.ignore("httplib")
    finder.ignore("md5")
    finder.ignore("new")
    finder.ignore("thread")
    finder.ignore("unittest2")
    finder.ignore("urllib2")
    finder.ignore("urlparse")

def hook_pycparser(finder, module):
    """pycparser needs lextab.py and yacctab.py which are not picked
    up automatically.  Make sure the complete package is included;
    otherwise the exe-files may create yacctab.py and lextab.py when
    they are run.
    """
    finder.import_package_later("pycparser")

def hook_pycparser__build_tables(finder, module):
    finder.ignore("lextab")
    finder.ignore("yacctab")
    finder.ignore("_ast_gen")
    finder.ignore("c_ast")

def hook_pycparser_ply(finder, module):
    finder.ignore("lex")
    finder.ignore("ply")

def hook_OpenSSL(finder, module):
    """OpenSSL needs the cryptography package."""
    finder.import_package_later("cryptography")

def hook_cffi_cparser(finder, module):
    finder.ignore("cffi._pycparser")

def hook_cffi(finder, module):
    # We need to patch two methods in the
    # cffi.vengine_cpy.VCPythonEngine class so that cffi libraries
    # work from within zip-files.
    finder.add_bootcode("""
def patch_cffi():
    def find_module(self, module_name, path, so_suffixes):
        import sys
        name = "%s.%s" % (self.verifier.ext_package, module_name)
        try:
            __import__(name)
        except ImportError:
            return None
        self.__module = mod = sys.modules[name]
        return mod.__file__

    def load_library(self):
        from cffi import ffiplatform
        import sys
        # XXX review all usages of 'self' here!
        # import it as a new extension module
        module = self.__module
        #
        # call loading_cpy_struct() to get the struct layout inferred by
        # the C compiler
        self._load(module, 'loading')
        #
        # the C code will need the <ctype> objects.  Collect them in
        # order in a list.
        revmapping = dict([(value, key)
                           for (key, value) in self._typesdict.items()])
        lst = [revmapping[i] for i in range(len(revmapping))]
        lst = list(map(self.ffi._get_cached_btype, lst))
        #
        # build the FFILibrary class and instance and call _cffi_setup().
        # this will set up some fields like '_cffi_types', and only then
        # it will invoke the chained list of functions that will really
        # build (notably) the constant objects, as <cdata> if they are
        # pointers, and store them as attributes on the 'library' object.
        class FFILibrary(object):
            _cffi_python_module = module
            _cffi_ffi = self.ffi
            _cffi_dir = []
            def __dir__(self):
                return FFILibrary._cffi_dir + list(self.__dict__)
        library = FFILibrary()
        if module._cffi_setup(lst, ffiplatform.VerificationError, library):
            import warnings
            warnings.warn("reimporting %r might overwrite older definitions"
                          % (self.verifier.get_module_name()))
        #
        # finally, call the loaded_cpy_xxx() functions.  This will perform
        # the final adjustments, like copying the Python->C wrapper
        # functions from the module to the 'library' object, and setting
        # up the FFILibrary class with properties for the global C variables.
        self._load(module, 'loaded', library=library)
        module._cffi_original_ffi = self.ffi
        module._cffi_types_of_builtin_funcs = self._types_of_builtin_functions
        return library

    from cffi.vengine_cpy import VCPythonEngine

    VCPythonEngine.find_module = find_module
    VCPythonEngine.load_library = load_library

patch_cffi()
del patch_cffi
""")

def hook_multiprocessing(finder, module):
    module.__globalnames__.add("AuthenticationError")
    module.__globalnames__.add("BufferTooShort")
    module.__globalnames__.add("Manager")
    module.__globalnames__.add("TimeoutError")
    module.__globalnames__.add("cpu_count")
    module.__globalnames__.add("current_process")
    module.__globalnames__.add("get_context")
    module.__globalnames__.add("get_start_method")
    module.__globalnames__.add("set_start_method")

    module.__globalnames__.add("JoinableQueue")
    module.__globalnames__.add("Lock")
    module.__globalnames__.add("Process")
    module.__globalnames__.add("Queue")
    module.__globalnames__.add("freeze_support")

def import_psutil(finder, module):
    """Exclude stuff for other operating systems."""
    finder.excludes.append("_psutil_bsd")
    finder.excludes.append("_psutil_linux")
    finder.excludes.append("_psutil_osx")
    finder.excludes.append("_psutil_posix")
    finder.excludes.append("_psutil_sunos")

def hook_PIL(finder, module):
    # c:\Python33-64\lib\site-packages\PIL
    """Pillow loads plugins"""
    # Exclude python 2 imports
    finder.excludes.append("Tkinter")
    finder.import_package_later("PIL")

def hook__socket(finder, module):
    """
    _socket.pyd uses the 'idna' encoding; and that requires
    'unicodedata.pyd'.
    """
    finder.import_hook("encodings.idna")
    finder.import_hook("unicodedata")

def hook_pyreadline(finder, module):
    """
    """
    finder.ignore("IronPythonConsole")
    finder.excludes.append("StringIO") # in pyreadline.py3k_compat
    finder.ignore("System")
    finder.excludes.append("sets")
    finder.ignore("startup")

def hook_xml_etree_ElementTree(finder, module):
    """ElementC14N is an optional extension. Ignore if it is not
    found.
    """
    finder.ignore("ElementC14N")

def hook_urllib_request(finder, module):
    """urllib.request imports _scproxy on darwin
    """
    finder.excludes.append("_scproxy")

def hook_pythoncom(finder, module):
    """pythoncom is a Python extension module with .dll extension,
    usually in the windows system directory as pythoncom3X.dll.
    """
    import pythoncom
    finder.add_dll(pythoncom.__file__)

def hook_pywintypes(finder, module):
    """pywintypes is a Python extension module with .dll extension,
    usually in the windows system directory as pywintypes3X.dll.
    """
    import pywintypes
    finder.add_dll(pywintypes.__file__)

def hook_win32com(finder, module):
    """The win32com package extends it's __path__ at runtime.
    """
    finder.import_hook("pywintypes")
    finder.import_hook("pythoncom")
    import win32com
    module.__path__ = win32com.__path__

def hook_win32api(finder, module):
    """win32api.FindFiles(...) needs this."""
    #finder.import_hook("pywintypes")
    finder.import_hook("win32timezone")

def hook_tkinter(finder, module):
    """Recusively copy tcl and tk directories.
    """
    # It probably doesn't make sense to exclude tix from the tcl distribution,
    # and only copy it when tkinter.tix is imported...
    import tkinter._fix as fix
    tcl_dir = os.path.normpath(os.path.join(fix.tcldir, ".."))
    assert os.path.isdir(tcl_dir)
    finder.add_datadirectory("tcl", tcl_dir, recursive=True)
    finder.set_min_bundle("tkinter", 2)

def hook_six(finder, module):
    """six.py has an object 'moves'. This allows to import
    modules/packages via attribute access under new names.

    We install a fake module named 'six.moves' which simulates this
    behaviour.
    """

    class SixImporter(type(module)):
        """Simulate six.moves.

        Import renamed modules when retrived as attributes.
        """

        __code__ = None

        def __init__(self, mf, *args, **kw):
            import six
            self.__moved_modules = {item.name: item.mod
                                    for item in six._moved_attributes
                                    if isinstance(item, six.MovedModule)}
            super().__init__(*args, **kw)
            self.__finder = mf

        def __getattr__(self, name):
            if name in self.__moved_modules:
                renamed = self.__moved_modules[name]
                self.__finder.safe_import_hook(renamed, caller=self)
                mod = self.__finder.modules[renamed]
                # add the module again with the renamed name:
                self.__finder._add_module("six.moves." + name, mod)
                return mod
            else:
                raise AttributeError(name)

    m = SixImporter(finder,
                    None, "six.moves", finder._optimize)
    finder._add_module("six.moves", m)

def hook_matplotlib(finder, module):
    """matplotlib requires data files in a 'mpl-data' subdirectory in
    the same directory as the executable.
    """
    # c:\Python33\lib\site-packages\matplotlib
    mpl_data_path = os.path.join(os.path.dirname(module.__loader__.path),
                                 "mpl-data")
    finder.add_datadirectory("mpl-data", mpl_data_path, recursive=True)
    finder.excludes.append("wx")
    # XXX matplotlib requires tkinter which modulefinder does not
    # detect because of the six bug.

def hook_numpy(finder, module):
    """numpy for Python 3 still tries to import some Python 2 modules;
    exclude them."""
    # I'm not sure if we can safely exclude these:
    finder.ignore("Numeric")
    finder.ignore("numarray")
    finder.ignore("numpy_distutils")
    finder.ignore("setuptools")
    finder.ignore("Pyrex")
    finder.ignore("nose")
    finder.ignore("scipy")

def hook_nose(finder, module):
    finder.ignore("IronPython")
    finder.ignore("cStringIO")
    finder.ignore("unittest2")

def hook_sysconfig(finder, module):
    finder.ignore("_sysconfigdata")

def hook_numpy_random_mtrand(finder, module):
    """the numpy.random.mtrand module is an extension module and the
    numpy.random module imports * from this module; define the list of
    global names available to this module in order to avoid spurious
    errors about missing modules.
    """
    module.__globalnames__.add('RandomState')
    module.__globalnames__.add('beta')
    module.__globalnames__.add('binomial')
    module.__globalnames__.add('bytes')
    module.__globalnames__.add('chisquare')
    module.__globalnames__.add('choice')
    module.__globalnames__.add('dirichlet')
    module.__globalnames__.add('exponential')
    module.__globalnames__.add('f')
    module.__globalnames__.add('gamma')
    module.__globalnames__.add('geometric')
    module.__globalnames__.add('get_state')
    module.__globalnames__.add('gumbel')
    module.__globalnames__.add('hypergeometric')
    module.__globalnames__.add('laplace')
    module.__globalnames__.add('logistic')
    module.__globalnames__.add('lognormal')
    module.__globalnames__.add('logseries')
    module.__globalnames__.add('multinomial')
    module.__globalnames__.add('multivariate_normal')
    module.__globalnames__.add('negative_binomial')
    module.__globalnames__.add('noncentral_chisquare')
    module.__globalnames__.add('noncentral_f')
    module.__globalnames__.add('normal')
    module.__globalnames__.add('np')
    module.__globalnames__.add('operator')
    module.__globalnames__.add('pareto')
    module.__globalnames__.add('permutation')
    module.__globalnames__.add('poisson')
    module.__globalnames__.add('power')
    module.__globalnames__.add('rand')
    module.__globalnames__.add('randint')
    module.__globalnames__.add('randn')
    module.__globalnames__.add('random_integers')
    module.__globalnames__.add('random_sample')
    module.__globalnames__.add('rayleigh')
    module.__globalnames__.add('seed')
    module.__globalnames__.add('set_state')
    module.__globalnames__.add('shuffle')
    module.__globalnames__.add('standard_cauchy')
    module.__globalnames__.add('standard_exponential')
    module.__globalnames__.add('standard_gamma')
    module.__globalnames__.add('standard_normal')
    module.__globalnames__.add('standard_t')
    module.__globalnames__.add('triangular')
    module.__globalnames__.add('uniform')
    module.__globalnames__.add('vonmises')
    module.__globalnames__.add('wald')
    module.__globalnames__.add('weibull')
    module.__globalnames__.add('zipf')

def hook_numpy_distutils(finder, module):
    """In a 'if sys.version_info[0] < 3:' block numpy.distutils does
    an implicit relative import: 'import __config__'.  This will not
    work in Python3 so ignore it.
    """
    finder.excludes.append("__config__")

def hook_numpy_f2py(finder, module):
    """ numpy.f2py tries to import __svn_version__.  Ignore when his fails.
    """
    finder.excludes.append("__svn_version__")

def hook_numpy_core_umath(finder, module):
    """the numpy.core.umath module is an extension module and the numpy module
       imports * from this module; define the list of global names available
       to this module in order to avoid spurious errors about missing
       modules"""
    module.__globalnames__.add("add")
    module.__globalnames__.add("absolute")
    module.__globalnames__.add("arccos")
    module.__globalnames__.add("arccosh")
    module.__globalnames__.add("arcsin")
    module.__globalnames__.add("arcsinh")
    module.__globalnames__.add("arctan")
    module.__globalnames__.add("arctanh")
    module.__globalnames__.add("bitwise_and")
    module.__globalnames__.add("bitwise_or")
    module.__globalnames__.add("bitwise_xor")
    module.__globalnames__.add("ceil")
    module.__globalnames__.add("conjugate")
    module.__globalnames__.add("cosh")
    module.__globalnames__.add("divide")
    module.__globalnames__.add("exp")
    module.__globalnames__.add("e")
    module.__globalnames__.add("fabs")
    module.__globalnames__.add("floor")
    module.__globalnames__.add("floor_divide")
    module.__globalnames__.add("fmod")
    module.__globalnames__.add("geterrobj")
    module.__globalnames__.add("greater")
    module.__globalnames__.add("hypot")
    module.__globalnames__.add("invert")
    module.__globalnames__.add("isfinite")
    module.__globalnames__.add("isinf")
    module.__globalnames__.add("isnan")
    module.__globalnames__.add("less")
    module.__globalnames__.add("left_shift")
    module.__globalnames__.add("log")
    module.__globalnames__.add("logical_and")
    module.__globalnames__.add("logical_not")
    module.__globalnames__.add("logical_or")
    module.__globalnames__.add("logical_xor")
    module.__globalnames__.add("maximum")
    module.__globalnames__.add("minimum")
    module.__globalnames__.add("multiply")
    module.__globalnames__.add("negative")
    module.__globalnames__.add("not_equal")
    module.__globalnames__.add("power")
    module.__globalnames__.add("remainder")
    module.__globalnames__.add("right_shift")
    module.__globalnames__.add("sign")
    module.__globalnames__.add("signbit")
    module.__globalnames__.add("sinh")
    module.__globalnames__.add("sqrt")
    module.__globalnames__.add("tan")
    module.__globalnames__.add("tanh")
    module.__globalnames__.add("true_divide")

def hook_numpy_core_numerictypes(finder, module):
    """the numpy.core.numerictypes module adds a number of items to itself
       dynamically; define these to avoid spurious errors about missing
       modules"""
    module.__globalnames__.add("bool_")
    module.__globalnames__.add("cdouble")
    module.__globalnames__.add("complexfloating")
    module.__globalnames__.add("csingle")
    module.__globalnames__.add("double")
    module.__globalnames__.add("longdouble")
    module.__globalnames__.add("float32")
    module.__globalnames__.add("float64")
    module.__globalnames__.add("float_")
    module.__globalnames__.add("inexact")
    module.__globalnames__.add("integer")
    module.__globalnames__.add("intc")
    module.__globalnames__.add("int32")
    module.__globalnames__.add("number")
    module.__globalnames__.add("single")

def hook_numpy_core(finder, module):
    finder.ignore("numpy.core._dotblas")


"""
========================================================
Compare Stochastic learning strategies for MLPClassifier
========================================================

This example visualizes some training loss curves for different stochastic
learning strategies, including SGD and Adam. Because of time-constraints, we
use several small datasets, for which L-BFGS might be more suitable. The
general trend shown in these examples seems to carry over to larger datasets,
however.
"""

print(__doc__)
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn import datasets

# different learning rate schedules and momentum parameters
params = [{'algorithm': 'sgd', 'learning_rate': 'constant', 'momentum': 0,
           'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'constant', 'momentum': .9,
           'nesterovs_momentum': False, 'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'constant', 'momentum': .9,
           'nesterovs_momentum': True, 'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'invscaling', 'momentum': 0,
           'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'invscaling', 'momentum': .9,
           'nesterovs_momentum': True, 'learning_rate_init': 0.2},
          {'algorithm': 'sgd', 'learning_rate': 'invscaling', 'momentum': .9,
           'nesterovs_momentum': False, 'learning_rate_init': 0.2},
          {'algorithm': 'adam'}]

labels = ["constant learning-rate", "constant with momentum",
          "constant with Nesterov's momentum",
          "inv-scaling learning-rate", "inv-scaling with momentum",
          "inv-scaling with Nesterov's momentum", "adam"]

plot_args = [{'c': 'red', 'linestyle': '-'},
             {'c': 'green', 'linestyle': '-'},
             {'c': 'blue', 'linestyle': '-'},
             {'c': 'red', 'linestyle': '--'},
             {'c': 'green', 'linestyle': '--'},
             {'c': 'blue', 'linestyle': '--'},
             {'c': 'black', 'linestyle': '-'}]

def plot_on_dataset(X, y, ax, name):
    # for each dataset, plot learning for each learning strategy
    print("\nlearning on dataset %s" % name)
    ax.set_title(name)
    X = MinMaxScaler().fit_transform(X)
    mlps = []
    if name == "digits":
        # digits is larger but converges fairly quickly
        max_iter = 15
    else:
        max_iter = 400

    for label, param in zip(labels, params):
        print("training: %s" % label)
        mlp = MLPClassifier(verbose=0, random_state=0,
                            max_iter=max_iter, **param)
        mlp.fit(X, y)
        mlps.append(mlp)
        print("Training set score: %f" % mlp.score(X, y))
        print("Training set loss: %f" % mlp.loss_)
    for mlp, label, args in zip(mlps, labels, plot_args):
            ax.plot(mlp.loss_curve_, label=label, **args)

fig, axes = plt.subplots(2, 2, figsize=(15, 10))
# load / generate some toy datasets
iris = datasets.load_iris()
digits = datasets.load_digits()
data_sets = [(iris.data, iris.target),
             (digits.data, digits.target),
             datasets.make_circles(noise=0.2, factor=0.5, random_state=1),
             datasets.make_moons(noise=0.3, random_state=0)]

for ax, data, name in zip(axes.ravel(), data_sets, ['iris', 'digits',
                                                    'circles', 'moons']):
    plot_on_dataset(*data, ax=ax, name=name)

fig.legend(ax.get_lines(), labels=labels, ncol=3, loc="upper center")
plt.show()


from mylib import *
import os,sys
import numpy as np
import matplotlib.pyplot as plt
import math
import random
from time import time

if __name__ == '__main__':
	DIR_PATH = sys.path[0] + '\\'

	# normal distribution vector file
	nvctr_file1 = DIR_PATH + 'normal_500_1.txt'
	nvctr_file2 = DIR_PATH + 'normal_500_2.txt'
	# uniform distribution vector file
	uvctr_file1 = DIR_PATH + 'uniform_500_1.txt'
	uvctr_file2 = DIR_PATH + 'uniform_500_2.txt'

	# normal distribution matrix
	nmtrx = fget_mtrx(nvctr_file1) + fget_mtrx(nvctr_file2)
	# uniform distribution matrix
	umtrx = fget_mtrx(uvctr_file1) + fget_mtrx(uvctr_file2)

	# plist is list the numbers of dimensions after DCT compression
	# nplist is for normal distribution data set
	# uplist is for uniform distribution data set
	nplist = []
	uplist = []
	for vector in nmtrx:
		u, p = my_DCT_compression(vector, 0.01)
		nplist.append(p)

	for vector in umtrx:
		u, p = my_DCT_compression(vector, 0.01)
		uplist.append(p)

	# draw histogram
	plt.figure(1)
	plt.subplot(2,1,1)
	my_hist(nplist, bucket_size = 1, flat_edge = False, title = "For normal distribution data set")
	plt.subplot(2,1,2)
	my_hist(uplist, bucket_size = 1, flat_edge = False, title = "For uniform distribution data set")
	plt.show()


# -*- coding: utf-8 -*-

# from framesplit import trajectory
# too slow using this module

import matplotlib.pyplot as plt

dirName = r"F:\simulations\asphaltenes\na-mont\TMBO-oil\water\373-continue/"
xyzName = 'all.xyz'
hetero = 'O'  # 'oh' 'N' 'sp'      'O' 'Np' 'sp'

with open(dirName + xyzName, 'r') as foo:
    coords = foo.readlines()
    nAtoms = int(coords[0])
    nFrames = int(len(coords) / (nAtoms + 2))
    pos = []
    for i in range(nFrames):
        istart = i * (nAtoms + 2)
        iend = (i + 1) * (nAtoms + 2)
        pos.append(coords[istart:iend])

# for i in range(200):
#     print coords[i]

heteroatom = 0
# all of my molecules have atoms less than 200
for i in range(200):
    x = pos[0][i].split()[0]
    if x == hetero:
        heteroatom = i
        break

heteroZ = []
for p in pos:
    # print p[heteroatom].split()[0]
    zx = float(p[heteroatom].split()[3])
    if zx < 10:
        zx = zx + 80
    heteroZ.append(zx)

with open(dirName + 'heteroZ.dat', 'w') as foo:
    for i, z in enumerate(heteroZ):
        print >> foo, "%3d %8.5f" % (i, z)

# energy plot
plt.figure(0, figsize=(8, 4))
figName = dirName + 'heteroZ.png'
plt.title('z of heteroatom', fontsize=20)
plt.plot(range(len(heteroZ)-1), heteroZ[1:], linewidth=2)
plt.grid(True)
plt.xlabel('steps')
plt.ylabel('Z')
plt.axis([0, len(heteroZ)*1.1, 0, max(heteroZ)*1.1])
plt.savefig(figName, format='png', dpi=300)
plt.close()


# Copyright 2017, Rohan Dandage <rraadd_8@hotmail.com,rohan@igib.in>
# This program is distributed under General Public License v. 3.

"""
================================
``io_mut_files``
================================
"""
from __future__ import division
import sys
from os.path import splitext,exists,basename,abspath,dirname
from os import makedirs,stat
import pandas as pd
import numpy as np
from glob import glob
import logging
import subprocess

from dms2dfe.lib.io_dfs import set_index

def getcolsmets(d,cs):
    """
    Get the total non-na items in column.

    :param d: pandas dataframe
    :param cs: list of columns
    """
    cols=[c for c in d.columns if cs in c]
    counts=[len(d[c].dropna()) for c in cols]
    return dict(zip(cols,counts))

def get_2SD_cutoffs(d,reps,N=False):
    """
    Get 2SD cutoffs for values in given column

    :param d: pandas dataframe
    :param reps: names of replicate conditions
    """
    t=d.copy()
    mus=[]
    sigmas=[]
    for r1i,r1 in enumerate(reps):
        for r2i,r2 in enumerate(reps):
            if r1i<r2i:
                t.loc[:,'reps']=t.loc[:,r1]-t.loc[:,r2]
                if N and ('mut' in t):
                    t.loc[(t.loc[:,'mut']==t.loc[:,'ref']),'reps']=np.nan
                mus.append(t.loc[:,'reps'].mean())
                sigmas.append(t.loc[:,'reps'].std())
    mu=np.mean(mus)
    sigma=np.sqrt(np.mean([s**2 for s in sigmas]))
    return mu+sigma*2,mu,sigma

def get_repli_FiA(d,csel='.NiA_tran.sel',cref='.NiA_tran.ref'):
    """
    Get replicates from data_fit

    :param d: pandas dataframe data_fit
    """
    sels=np.sort([c for c in d.columns if csel in c])
    refs=np.sort([c for c in d.columns if cref in c])
    FCAi=1
    cols_FCA=[]
    for refi,ref in enumerate(refs):
        for seli,sel in enumerate(sels):
            if refi==seli:
                coln='FCA%s_reps' % FCAi
                d.loc[:,coln]=d.loc[:,sel]-d.loc[:,ref]
                d.loc[(d.loc[:,'mut']==d.loc[:,'ref']),coln]=np.nan
                cols_FCA.append(coln)
                FCAi+=1
    return d.loc[:,cols_FCA]

def data_fit2cutoffs(d,sA,sB,N=True):
    """
    Get cutoffs of data fit to assign classes

    :param d: pandas dataframe with fold change values
    :param sA,sB: columns with two conditions to be compared
    """
    refs=[c for c in d.columns if sA in c]
    sels=[c for c in d.columns if sB in c]
    _,mu1,sigma1=get_2SD_cutoffs(d,refs)
    _,mu2,sigma2=get_2SD_cutoffs(d,sels)
    return np.mean([mu1,mu2])+2*np.sqrt(np.mean([sigma1,sigma2]))

def class_fit(d,col_fit='FiA',FC=True,zscore=False): #column of the data_fit
    """
    This classifies the fitness of mutants into beneficial, neutral or, deleterious.

    :param d: dataframe of `data_fit`.
    :returns d: classes of fitness written in 'class-fit' column based on values in column 'FiA'.
    """
    cols_reps=[c for c in d if '.NiA_tran.ref' in c]
    if FC and (len(cols_reps)==2):
        up,_,_=get_2SD_cutoffs(d,cols_reps,N=True)
        dw=-1*up
    else:
        if zscore:
            up,dw=-2,2
        else:
            up,dw=0,0
    d.loc[d.loc[:,col_fit]>+up,    'class_fit']="enriched"
    d.loc[((d.loc[:,col_fit]>=dw) & (d.loc[:,col_fit]<=up)),'class_fit']="neutral"
    d.loc[d.loc[:,col_fit]<dw,    'class_fit']="depleted"
    return d

def class_comparison(dA,dB):
    """
    This classifies differences in fitness i.e. relative fitness into positive, negative or robust categories.

    :param dc: dataframe with `dc`.
    :returns dc: dataframe with `class__comparison` added according to fitness levels in input and selected samples in `dc`
    """
    dA=set_index(dA,'mutids')
    dB=set_index(dB,'mutids')
    dc=get_repli_FiA(dA).join(get_repli_FiA(dB),lsuffix='_test',rsuffix='_ctrl')
    up=data_fit2cutoffs(dc,sA='_reps_test',sB='_reps_ctrl',N=False)
    dw=-1*up

    diff=dA.loc[:,'FiA']-dB.loc[:,'FiA']
    diff.index.name='mutids'
    diff=diff.reset_index()
    mutids_up=diff.loc[(diff.loc[:,'FiA']>up),'mutids'].tolist()
    mutids_dw=diff.loc[(diff.loc[:,'FiA']<dw),'mutids'].tolist()

    dc.loc[mutids_up,'class_comparison']="positive"
    dc.loc[mutids_dw,'class_comparison']="negative"
    return dc.loc[:,'class_comparison']

def get_data_metrics(prj_dh):
    """
    Obtain metrics for fold change values from a experiment

    :param prj_dh: path to project directory
    """
    data_fit_fns_all=[basename(s) for s in glob('%s/data_fit/aas/*' % prj_dh)]
    data_fit_fns_all2labels=dict(zip(data_fit_fns_all,data_fit_fns_all))
    data_fit_metrics=pd.DataFrame(index=data_fit_fns_all)

    fit=pd.read_csv('%s/cfg/fit' % prj_dh).set_index('unsel')
    comparison=pd.read_csv('%s/cfg/comparison' % prj_dh).set_index('ctrl')

    for ctrl in comparison.index:
        fh_ctrls=glob('%s/data_fit/aas/%s_WRT_*' % (prj_dh,ctrl))
        for fh_ctrl in fh_ctrls:
            dctrl=pd.read_csv(fh_ctrl).set_index('mutids')

            up,_,_=get_2SD_cutoffs(dctrl,[c for c in dctrl if '.NiA_tran.ref' in c],N=True)
            dw=-1*up
            # print up,dw
            ctrl=basename(fh_ctrl)
            ctrls={'versus_%s' % ctrl:ctrl}

            data_fit_metrics.index.name='fn'
            for fni,fn in enumerate(data_fit_fns_all):
                fh= '%s/data_fit/aas/%s' % (prj_dh,fn)
                d=pd.read_csv(fh).set_index('mutids')
                data_fit_metrics.loc[fn,'$\mu+2\sigma$']=up
                d=d.reset_index()
                mutids_up=d.loc[(d.loc[:,'FiA']>up),'mutids'].tolist()
                mutids_dw=d.loc[(d.loc[:,'FiA']<dw),'mutids'].tolist()
                mutids_updw=mutids_up+mutids_dw
                d=d.set_index('mutids')
                data_fit_metrics.loc[fn,'$n$']=len(d.loc[:,'FiA'].dropna())
                ref_counts=getcolsmets(d,cs='.NiA_tran.ref')
                sel_counts=getcolsmets(d,cs='.NiA_tran.sel')
                for i,_ in enumerate(ref_counts.keys()):
                    data_fit_metrics.loc[fn,'$n$ ref%02d' % i]=ref_counts[ref_counts.keys()[i]]
                for i,_ in enumerate(sel_counts.keys()):
                    data_fit_metrics.loc[fn,'$n$ sel%02d' % i]=sel_counts[sel_counts.keys()[i]]
                data_fit_metrics.loc[fn,'$n_{neutral}$']=data_fit_metrics.loc[fn,'$n$']-len(mutids_updw)
                data_fit_metrics.loc[fn,'$n_{enriched}$']=len(mutids_up)
                data_fit_metrics.loc[fn,'$n_{depleted}$']=len(mutids_dw)
                data_fit_metrics.loc[fn,'$F$ mean']=d.loc[:,'FiA'].mean()

    if len(data_fit_metrics.columns)>0:
        data_fit_metrics['$n_{enriched}$%%']=data_fit_metrics['$n_{enriched}$']\
        /(data_fit_metrics['$n_{enriched}$']+data_fit_metrics['$n_{depleted}$'])*100
        data_fit_metrics['$n_{depleted}$%%']=data_fit_metrics['$n_{depleted}$']\
        /(data_fit_metrics['$n_{enriched}$']+data_fit_metrics['$n_{depleted}$'])*100
        data_fit_metrics['$n_{enriched}$%']=data_fit_metrics['$n_{enriched}$']/data_fit_metrics['$n$']*100
        data_fit_metrics['$n_{depleted}$%']=data_fit_metrics['$n_{depleted}$']/data_fit_metrics['$n$']*100
        data_fit_metrics['$n_{neutral}$%']=data_fit_metrics['$n_{neutral}$']/data_fit_metrics['$n$']*100

        data_fit_metrics['$E$']=data_fit_metrics['$n_{enriched}$']/\
        (data_fit_metrics['$n_{enriched}$']+data_fit_metrics['$n_{depleted}$'])
        # print data_fit_metrics.index
        # print data_fit_metrics.columns

        data_fit_metrics.loc[:,'labels']=[data_fit_fns_all2labels[i] for i in data_fit_metrics.index]

        for c in ctrls:
            fh= '%s/data_fit/aas/%s' % (prj_dh,ctrls[c])
            dA=pd.read_csv(fh).set_index('mutids')
            for fni,fn in enumerate(data_fit_fns_all):
                fh= '%s/data_fit/aas/%s' % (prj_dh,fn)
                dB=pd.read_csv(fh).set_index('mutids')
                dc=get_repli_FiA(dA).join(get_repli_FiA(dB),lsuffix='_ctrl',rsuffix='_test')
                up=data_fit2cutoffs(dc,sA='_reps_test',sB='_reps_ctrl',N=False)
                dw=-1*up
                data_fit_metrics.loc[fn,'$\mu+2\sigma$ comparison %s' % c]=up

                diff=dB.loc[:,'FiA']-dA.loc[:,'FiA']
                diff=diff.reset_index()
                mutids_up=diff.loc[(diff.loc[:,'FiA']>up),'mutids'].tolist()
                mutids_dw=diff.loc[(diff.loc[:,'FiA']<dw),'mutids'].tolist()
                mutids_updw=mutids_up+mutids_dw
                diff=diff.set_index('mutids')
                data_fit_metrics.loc[fn,'$n$ %s' % c]=len(diff.dropna())
                data_fit_metrics.loc[fn,'$n_{positive}$ %s' % c]=len(mutids_up)
                data_fit_metrics.loc[fn,'$n_{negative}$ %s' % c]=len(mutids_dw)

            data_fit_metrics['$\Delta n$ %s' % c]=(data_fit_metrics['$n$']-data_fit_metrics.loc[ctrls[c],'$n$'])
            data_fit_metrics['$\Delta n_{enriched}$ %s' % c]=(data_fit_metrics['$n_{enriched}$']-data_fit_metrics.loc[ctrls[c],'$n_{enriched}$'])
            data_fit_metrics['$\Delta n_{enriched}$%% %s' % c]=(data_fit_metrics['$n_{enriched}$%']-data_fit_metrics.loc[ctrls[c],'$n_{enriched}$%'])
            data_fit_metrics['$\Delta F$ %s' % c]=(data_fit_metrics['$F$ mean']-data_fit_metrics.loc[ctrls[c],'$F$ mean'])
        data_fit_metrics_fh='%s/data_comparison/data_fit_metrics' % prj_dh
        data_fit_metrics.to_csv(data_fit_metrics_fh)
        return data_fit_metrics
    else:
        logging.info('data_metrics has 0 cols')


import numpy as np
import matplotlib.pylab as plt
from MulLayer import MulLayer

class AddLayer:
        def __init__ (self):
                pass

        def forward(self, x, y):
                out = x + y
                return out

        def backward(self, dout):
                dx = dout * 1
                dy = dout * 1
                return dx,dy

def test_add_layer():
        apple = 100
        apple_num = 2
        orange = 150
        orange_num = 3
        tax = 1.1

        mul_apple_layer = MulLayer()
        mul_orange_layer = MulLayer()
        add_apple_orange_layer = AddLayer()
        mul_tax_layer = MulLayer()

        apple_price = mul_apple_layer.forward( apple, apple_num)
        orange_price = mul_orange_layer.forward( orange, orange_num)
        all_price = add_apple_orange_layer.forward( apple_price, orange_price)
        price = mul_tax_layer.forward( all_price, tax)

        dprice = 1
        dall_price, dtax = mul_tax_layer.backward( dprice)
        dapple_price, dorange_price = add_apple_orange_layer.backward(dall_price)
        dorange, dorange_num = mul_orange_layer.backward(dorange_price)
        dapple, dapple_num = mul_apple_layer.backward( dapple_price)

        print("price=", price)
        print(dapple_num, dapple, dorange, dorange_num, dtax)


# -*- coding: utf-8 -*-
"""
DBSCAN Acclerated by Facebook AI Faiss
DBSCAN: Density-Based Spatial Clustering of Applications with Noise
"""

# Author: Robert Layton <robertlayton@gmail.com>
#         Joel Nothman <joel.nothman@gmail.com>
#         Lars Buitinck
#
# License: BSD 3 clause

import numpy as np
import time
from scipy import sparse
from numba import autojit
import numba

from sklearn.base import BaseEstimator, ClusterMixin
from sklearn.utils import check_array, check_consistent_length
#from sklearn.neighbors import NearestNeighbors

from sklearn.cluster._dbscan_inner import dbscan_inner
import faiss

@autojit
def get_neighborhoods(D, I, eps):
    neighborhoods = []
    for i in range(len(D)):
        distances = D[i]
        #print(distances)
        distances = np.delete(distances, 0)
        indices = I[i]
        indices = np.delete(indices, 0)
        #print(indices)
        index = indices[distances <= eps]
        neighborhoods.append(index)
    #neighborhoods = np.asarray(neighborhoods)
    #np.savetxt('faiss_neighborhoods', np.asarray(neighborhoods), fmt='%s')
    return np.asarray(neighborhoods)

def cpu_radius_neighbors(X, eps, min_samples, nlist, nprobe, return_distance=False, IVFFlat=True):
    dimension = X.shape[1]
    if IVFFlat is True:
        quantizer = faiss.IndexFlatL2(dimension)
        index_cpu = faiss.IndexIVFFlat(quantizer, dimension, nlist, faiss.METRIC_L2)
        # here we specify METRIC_L2, by default it performs inner-product search
        assert not index_cpu.is_trained
        index_cpu.train(X)
        assert index_cpu.is_trained
        # here we specify METRIC_L2, by default it performs inner-product search
    else:
        index_cpu = faiss.IndexFlatL2(dimension)
    index_cpu.add(X)
    n_samples = 1000
    k = min_samples
    samples = np.random.choice(len(X), n_samples)
    # print(samples)
    D, I = index_cpu.search(X[samples], k)  # sanity check
    while np.min(np.amax(D, axis=1)) < eps:
        k = k * 2
       # D, I = index_gpu.search(X[samples], k)
        #print(np.min(np.amax(D, axis=1)), eps, k)
        D, I = index_cpu.search(X[samples], k)
    if k > 1024:
        k = 1000
        #print(np.max(D[:, k - 1]), k, eps)
    index_cpu.nprobe = nprobe
    D, I = index_cpu.search(X, k)  # actual search

    return get_neighborhoods(D, I, eps)

def gpu_radius_neighbors(X, eps, min_samples, nlist, nprobe, return_distance=False, IVFFlat=True):
    dimension = X.shape[1]
    if IVFFlat is True:
        quantizer = faiss.IndexFlatL2(dimension)
        index_cpu = faiss.IndexIVFFlat(quantizer, dimension, nlist, faiss.METRIC_L2)
        # here we specify METRIC_L2, by default it performs inner-product search
        res = faiss.StandardGpuResources() # use a single GPU
        flat_config = faiss.GpuIndexFlatConfig()
        flat_config.device = 0
        # make it an IVF GPU index
        index_gpu = faiss.index_cpu_to_gpu(res, 0, index_cpu)
        assert not index_gpu.is_trained
        index_gpu.train(X)
        assert index_gpu.is_trained
        # here we specify METRIC_L2, by default it performs inner-product search
    else:
        index_cpu = faiss.IndexFlatL2(dimension)
        res = faiss.StandardGpuResources()
        flat_config = faiss.GpuIndexFlatConfig()
        flat_config.device = 0
        index_gpu = faiss.index_cpu_to_gpu(res, 0, index_cpu)
    index_gpu.add(X)
    n_samples = 1000
    k = min_samples
    samples = np.random.choice(len(X), n_samples)
    # print(samples)
    D, I = index_gpu.search(X[samples], k)  # sanity check
    while np.max(D[:, k - 1]) < eps:
        k = k * 2
        D, I = index_gpu.search(X[samples], k)
        #print(np.max(D[:, k - 1]), k, eps)
    index_gpu.nprobe = nprobe
    D, I = index_gpu.search(X, k)  # actual search

    return get_neighborhoods(D, I, eps)

def faiss_dbscan(X, eps=0.5, min_samples=5, nlist=100, nprobe=5, metric='l2', metric_params=None,
           algorithm='auto', leaf_size=30, p=2, sample_weight=None, n_jobs=1, GPU=False, IVFFlat=True):
    """Perform DBSCAN clustering from vector array or distance matrix.

    Read more in the :ref:`User Guide <dbscan>`.

    Parameters
    ----------
    X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
            array of shape (n_samples, n_samples)
        A feature array, or array of distances between samples if
        ``metric='precomputed'``.

    eps : float, optional
        The maximum distance between two samples for them to be considered
        as in the same neighborhood.

    min_samples : int, optional
        The number of samples (or total weight) in a neighborhood for a point
        to be considered as a core point. This includes the point itself.

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.pairwise_distances for its
        metric parameter.
        If metric is "precomputed", X is assumed to be a distance matrix and
        must be square. X may be a sparse matrix, in which case only "nonzero"
        elements may be considered neighbors for DBSCAN.

    metric_params : dict, optional
        Additional keyword arguments for the metric function.

        .. versionadded:: 0.19

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        The algorithm to be used by the NearestNeighbors module
        to compute pointwise distances and find nearest neighbors.
        See NearestNeighbors module documentation for details.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or cKDTree. This can affect the speed
        of the construction and query, as well as the memory required
        to store the tree. The optimal value depends
        on the nature of the problem.

    p : float, optional
        The power of the Minkowski metric to be used to calculate distance
        between points.

    sample_weight : array, shape (n_samples,), optional
        Weight of each sample, such that a sample with a weight of at least
        ``min_samples`` is by itself a core sample; a sample with negative
        weight may inhibit its eps-neighbor from being core.
        Note that weights are absolute, and default to 1.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Returns
    -------
    core_samples : array [n_core_samples]
        Indices of core samples.

    labels : array [n_samples]
        Cluster labels for each point.  Noisy samples are given the label -1.

    Notes
    -----
    See examples/cluster/plot_dbscan.py for an example.

    This implementation bulk-computes all neighborhood queries, which increases
    the memory complexity to O(n.d) where d is the average number of neighbors,
    while original DBSCAN had memory complexity O(n).

    Sparse neighborhoods can be precomputed using
    :func:`NearestNeighbors.radius_neighbors_graph
    <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>`
    with ``mode='distance'``.

    References
    ----------
    Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based
    Algorithm for Discovering Clusters in Large Spatial Databases with Noise".
    In: Proceedings of the 2nd International Conference on Knowledge Discovery
    and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
    """
    if not eps > 0.0:
        raise ValueError("eps must be positive.")

    if sample_weight is not None:
        sample_weight = np.asarray(sample_weight)
        check_consistent_length(X, sample_weight)

    # Calculate neighborhood for all samples. This leaves the original point
    # in, which needs to be considered later (i.e. point i is in the
    # neighborhood of point i. While True, its useless information)
    if GPU is True:
        neighborhoods = gpu_radius_neighbors(X, eps, min_samples, nlist, nprobe, return_distance=False, IVFFlat=IVFFlat)
    else:
        neighborhoods = cpu_radius_neighbors(X, eps, min_samples, nlist, nprobe, return_distance=False, IVFFlat=IVFFlat)
    if sample_weight is None:
        n_neighbors = np.array([len(neighbors)
                                for neighbors in neighborhoods])
    else:
        n_neighbors = np.array([np.sum(sample_weight[neighbors])
                                for neighbors in neighborhoods])

    # Initially, all samples are noise.
    labels = -np.ones(X.shape[0], dtype=np.intp)

    # A list of all core samples found.
    core_samples = np.asarray(n_neighbors >= min_samples, dtype=np.uint8)
    dbscan_inner(core_samples, neighborhoods, labels)
    return np.where(core_samples)[0], labels

class Faiss_DBSCAN(BaseEstimator, ClusterMixin):
    """Perform DBSCAN clustering from vector array or distance matrix.

    DBSCAN - Density-Based Spatial Clustering of Applications with Noise.
    Finds core samples of high density and expands clusters from them.
    Good for data which contains clusters of similar density.

    Read more in the :ref:`User Guide <dbscan>`.

    Parameters
    ----------
    eps : float, optional
        The maximum distance between two samples for them to be considered
        as in the same neighborhood.

    min_samples : int, optional
        The number of samples (or total weight) in a neighborhood for a point
        to be considered as a core point. This includes the point itself.

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.calculate_distance for its
        metric parameter.
        If metric is "precomputed", X is assumed to be a distance matrix and
        must be square. X may be a sparse matrix, in which case only "nonzero"
        elements may be considered neighbors for DBSCAN.

        .. versionadded:: 0.17
           metric *precomputed* to accept precomputed sparse matrix.

    metric_params : dict, optional
        Additional keyword arguments for the metric function.

        .. versionadded:: 0.19

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        The algorithm to be used by the NearestNeighbors module
        to compute pointwise distances and find nearest neighbors.
        See NearestNeighbors module documentation for details.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or cKDTree. This can affect the speed
        of the construction and query, as well as the memory required
        to store the tree. The optimal value depends
        on the nature of the problem.

    p : float, optional
        The power of the Minkowski metric to be used to calculate distance
        between points.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run.
        If ``-1``, then the number of jobs is set to the number of CPU cores.

    Attributes
    ----------
    core_sample_indices_ : array, shape = [n_core_samples]
        Indices of core samples.

    components_ : array, shape = [n_core_samples, n_features]
        Copy of each core sample found by training.

    labels_ : array, shape = [n_samples]
        Cluster labels for each point in the dataset given to fit().
        Noisy samples are given the label -1.

    Notes
    -----
    See examples/cluster/plot_dbscan.py for an example.

    This implementation bulk-computes all neighborhood queries, which increases
    the memory complexity to O(n.d) where d is the average number of neighbors,
    while original DBSCAN had memory complexity O(n).

    Sparse neighborhoods can be precomputed using
    :func:`NearestNeighbors.radius_neighbors_graph
    <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>`
    with ``mode='distance'``.

    References
    ----------
    Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based
    Algorithm for Discovering Clusters in Large Spatial Databases with Noise".
    In: Proceedings of the 2nd International Conference on Knowledge Discovery
    and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
    """

    def __init__(self, eps=0.5, min_samples=5, nlist=100, nprobe=5, metric='l2', n_jobs=1, GPU=False, IVFFlat=True):
        self.eps = eps
        self.min_samples = min_samples
        self.metric = metric
        self.n_jobs = n_jobs
        self.GPU = GPU
        self.IVFFlat = IVFFlat
        self.nlist = nlist
        self.nprobe = nprobe

    def fit(self, X, y=None, sample_weight=None):
        """Perform DBSCAN clustering from features or distance matrix.

        Parameters
        ----------
        X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
                array of shape (n_samples, n_samples)
            A feature array, or array of distances between samples if
            ``metric='precomputed'``.
        sample_weight : array, shape (n_samples,), optional
            Weight of each sample, such that a sample with a weight of at least
            ``min_samples`` is by itself a core sample; a sample with negative
            weight may inhibit its eps-neighbor from being core.
            Note that weights are absolute, and default to 1.
        """
        #if metric is not "rmsd":
        #    X = check_array(X, accept_sparse='csr')
        #t0 = time.time()
        clust = faiss_dbscan(X, eps=self.eps, min_samples=self.min_samples, nlist=self.nlist, nprobe=self.nprobe, sample_weight=sample_weight, GPU=self.GPU, IVFFlat=self.IVFFlat)
        #t1 = time.time()
        #print("Faiss DBSCAN clustering Time Cost:", t1 - t0)
        self.core_sample_indices_, self.labels_ = clust
        if len(self.core_sample_indices_):
            # fix for scipy sparse indexing issue
            self.components_ = X[self.core_sample_indices_].copy()
        else:
            # no core samples
            self.components_ = np.empty((0, X.shape[1]))
        return self


from analysis.preparation import labelMatrixToArray
from analysis.preparation import normalizeZeroClassArray
from visualise.trace_features import trace_feature_origin
from visualise.confusion_matrix import plot_confusion_matrix

import numpy as np
import sklearn
import sklearn.linear_model
import sklearn.preprocessing as pp
import sklearn.svm as svm
import sklearn.feature_selection as fs
from analysis.classification import fit_classifier
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt

# Interesting References:
# RFECV:
# Guyon, I., Weston, J., Barnhill, S., & Vapnik, V. (2002). Gene selection for
# cancer classification using support vector machines. Mach. Learn.. 46(1-3). 389-422.

def feature_selection(train_data, train_labels, const):
    train_labels_arr, exclude = labelMatrixToArray(train_labels, const.label_threshold)
    train_data_clean = train_data.drop(exclude)
    train_labels_arr, train_data_clean, _ = normalizeZeroClassArray(train_labels_arr, train_data_clean)

    print "num features before selection: {}".format(train_data_clean.columns.size)
    feature_index = variance_threshold(train_data_clean)
    clf, clf_name, needs_scaling = fit_classifier(train_data_clean.values[:,feature_index], np.array(train_labels_arr))
    prediction = clf.predict(get_values(train_data_clean, feature_index, needs_scaling))
    print("report for {} after variance threshold".format(clf_name))
    print(classification_report(train_labels_arr,prediction))
    cnf_matrix = confusion_matrix(train_labels_arr, prediction)
    plt.figure()
    plot_confusion_matrix(cnf_matrix, classes=['0.0','1.0','2.0','3.0','4.0','5.0','6.0','7.0'],
                          title="Confusion Matrix for {} after variance threshold".format(clf_name))
    trace_feature_origin(feature_index,const)

    feature_index = rfe(train_data_clean,train_labels_arr)
    clf, clf_name, needs_scaling = fit_classifier(train_data_clean.values[:, feature_index], np.array(train_labels_arr))
    prediction = clf.predict(get_values(train_data_clean, feature_index, needs_scaling))
    print("report for {} after RFE".format(clf_name))
    print(classification_report(train_labels_arr, prediction))
    cnf_matrix = confusion_matrix(train_labels_arr, prediction)
    plt.figure()
    plot_confusion_matrix(cnf_matrix, classes=['0.0','1.0','2.0','3.0','4.0','5.0','6.0','7.0'],
                          title="Confusion Matrix for {} after variance threshold".format(clf_name))
    trace_feature_origin(feature_index, const)

    feature_index = k_best_chi2(train_data_clean, train_labels_arr, 700)
    clf, clf_name, needs_scaling = fit_classifier(train_data_clean.values[:, feature_index], np.array(train_labels_arr))
    prediction = clf.predict(get_values(train_data_clean, feature_index, needs_scaling))
    print("report for {} after Chi2".format(clf_name))
    print(classification_report(train_labels_arr, prediction))
    cnf_matrix = confusion_matrix(train_labels_arr, prediction)
    plt.figure()
    plot_confusion_matrix(cnf_matrix, classes=['0.0','1.0','2.0','3.0','4.0','5.0','6.0','7.0'],
                          title="Confusion Matrix for {} after variance threshold".format(clf_name))
    trace_feature_origin(feature_index, const)

    feature_index = rfe_cv_f1(train_data_clean, train_labels_arr)
    clf, clf_name, needs_scaling = fit_classifier(train_data_clean.values[:, feature_index], np.array(train_labels_arr))
    prediction = clf.predict(get_values(train_data_clean, feature_index, needs_scaling))
    print("report for {} after RFECV".format(clf_name))
    print(classification_report(train_labels_arr, prediction))
    cnf_matrix = confusion_matrix(train_labels_arr, prediction)
    plt.figure()
    plot_confusion_matrix(cnf_matrix, classes=['0.0','1.0','2.0','3.0','4.0','5.0','6.0','7.0'],
                          title="Confusion Matrix for {} after variance threshold".format(clf_name))
    trace_feature_origin(feature_index, const)

    plt.show()

def get_values(data, feature_index, needs_scaling):
    if needs_scaling:
        values = data.values[:, feature_index]
        minmax = pp.MinMaxScaler()
        values = minmax.fit_transform(values)
        return values
    else:
        return data.values[:, feature_index]

def variance_threshold(train_data):
    # feature selection using VarianceThreshold filter
    sel = fs.VarianceThreshold(threshold=(.8 * (1 - .8)))
    fit = sel.fit(train_data.values)
    col_index = fit.get_support(indices=True)
    print "num features selected by VarianceThreshold: {}".format(len(col_index))
    return col_index

def rfe(train_data, train_labels):
    # important toto!
    # todo: I think also for feature selection we should take care the 0 class is balanced!
    # todo: if you use it that way, scale the features
    print "Recursive eleminate features: "
    svc = sklearn.linear_model.Lasso(alpha = 0.1) #svm.SVR(kernel="linear")
    print "scale data"
    values = train_data.values
    minmax = pp.MinMaxScaler()
    values = minmax.fit_transform(values)  # pp.scale(values)
    print "test fit."
    svc.fit(values, np.array(train_labels))
    print "run rfecv.."
    rfecv = fs.RFE(estimator=svc, step=0.1, verbose=2)
    rfecv.fit(values, np.array(train_labels))
    print "get support..."
    col_index = rfecv.get_support(indices=True)
    print "num features selected by RFE(CV)/Lasso: {}".format(len(col_index))
    return col_index

def rfe_cv_f1(train_data, train_labels):
    # important toto!
    # todo: I think also for feature selection we should take care the 0 class is balanced!
    # todo: if you use it that way, scale the features
    print "Recursive eleminate features: "
    svc = svm.SVC(kernel="linear") #sklearn.linear_model.Lasso(alpha = 0.1)
    print "scale data"
    values = train_data.values
    minmax = pp.MinMaxScaler()
    values = minmax.fit_transform(values)#pp.scale(values)
    print "test fit."
    svc.fit(values, np.array(train_labels).astype(int))
    print "run rfecv.."
    rfecv = fs.RFECV(estimator=svc, step=0.05, verbose=2)
    rfecv.fit(values, np.array(train_labels).astype(int))
    print "get support..."
    col_index = rfecv.get_support(indices=True)

    print "num features selected by RFECV/SVR: {}".format(len(col_index))
    return col_index

def k_best_chi2(train_data, train_labels, k):
    values = train_data.values
    if values.min() < 0:
        values = values + abs(values.min())
    kb = fs.SelectKBest(fs.chi2, k=k)
    kb.fit(values, np.array(train_labels))
    col_index = kb.get_support(indices=True)
    print "num features selected by K-Best using chi2: {}".format(len(col_index))
    return col_index


import numpy as np
import matplotlib.pyplot as plt
import essentia.standard as ess

M = 1024
N = 1024
H = 512
fs = 44100
spectrum = ess.Spectrum(size=N)
window = ess.Windowing(size=M, type='hann')
centroid = ess.Centroid(range=fs/2.0)
x = ess.MonoLoader(filename = '../../../sounds/speech-male.wav', sampleRate = fs)()
centroids = []

for frame in ess.FrameGenerator(x, frameSize=M, hopSize=H, startFromZero=True):
  mX = spectrum(window(frame))
  centroid_val = centroid(mX)
  centroids.append(centroid_val)
centroids = np.array(centroids)

plt.figure(1, figsize=(9.5, 5))
plt.subplot(2,1,1)

plt.plot(np.arange(x.size)/float(fs), x)
plt.axis([0, x.size/float(fs), min(x), max(x)])
plt.ylabel('amplitude')
plt.title('x (speech-male.wav)')

plt.subplot(2,1,2)
frmTime = H*np.arange(centroids.size)/float(fs)
plt.plot(frmTime, centroids, 'g', lw=1.5)
plt.axis([0, x.size/float(fs), min(centroids), max(centroids)])
plt.xlabel('time (sec)')
plt.ylabel('frequency (Hz)')
plt.title('spectral centroid')

plt.tight_layout()
plt.savefig('centroid.png')
plt.show()


import numpy as np
import matplotlib.pyplot as plt
import os
from os.path import join
import cv2
from skimage.transform import resize
from tqdm import tqdm
from datetime import datetime
import functools

os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

from keras import optimizers
from keras.models import Model, load_model
from keras.layers import Flatten, Dense, Conv2D, Dropout, MaxPooling2D, BatchNormalization, Input
from keras.callbacks import ModelCheckpoint, Callback
import json
import tensorflow as tf

tf.reset_default_graph()
from keras import backend as K

K.image_dim_ordering()

###################### Hyperparameters ######################
#                    Mode paramaters
INPUT_SHAPE = (256, 256, 3)
IMAGE_NUMBER = 30000
WEIGHT_FILENAME = os.path.join('models', 'tag_prediction.hdf5')
ROOT = 'dress'
VALIDATION_SPLIT = 0.9
TAG_END = "_dress"
FILE_END = 'S'
MIN_TAG_USE = 500
#                    Training parameters
BATCH_SIZE = 8
EPOCHS = 100
LEARNING_RATE = 1 * 10 ** -3
DROPOUT = 0.5
MOMENTUM = 0.5
WEIGHT_DECAY = 4 * 10 ** -5  # weight decay
ACTIVATION = 'selu'
NEURON_BASIS = 32

class PlotLearning(Callback):
    def __init__(self, examples=False):
        super().__init__()
        self.examples = examples
        self.x = []
        self.losses, self.val_losses = [], []
        self.logs = []

    def on_epoch_end(self, epoch, logs={}):
        self.logs.append(logs)
        self.x.append(epoch)
        self.losses.append(logs.get('loss'))
        self.val_losses.append(logs.get('val_loss'))

        plt.yscale('log')
        plt.plot(self.x, self.losses)
        plt.plot(self.x, self.val_losses)
        plt.xlabel('Epochs')
        plt.ylabel('Crossentropy')
        plt.legend(['Training', 'Validation'])

        plt.tight_layout()
        plt.savefig('plot.png')
        plt.close()

        if self.examples:
            z = self.model.predict(self.model.example[0][:6])
            plot_example(self.model.example[0][:6], self.model.example[1][:6], z, self.model.labels)
            plt.savefig(f"epochs/epoch{self.x[-1]}.png")
            plt.close()

def plot_example(xs, ys, zs, labels):
    n = xs.shape[0]
    plt.figure(figsize=(12, 8))
    plt.tight_layout()
    for i, (x, y, z) in enumerate(zip(xs, ys, zs)):
        if i != 0:
            tick_label = [' ' for label in labels]
        else:
            tick_label = labels
        plt.subplot(3, n, i + 1)
        plt.imshow(x)
        plt.subplot(3, n, i + n + 1)
        plt.barh(labels, y, tick_label=tick_label)
        plt.xlim(0, 1)
        plt.subplot(3, n, i + 2 * n + 1)
        plt.barh(labels, z, tick_label=tick_label)
        plt.xlim(0, 1)

def import_model(tag_number, input_shape=INPUT_SHAPE):
    inputs = Input(input_shape)

    layer = Conv2D(NEURON_BASIS, (3, 3), activation=ACTIVATION, padding='same')(inputs)
    layer = Conv2D(NEURON_BASIS, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = MaxPooling2D(pool_size=(2, 2))(layer)

    layer = Conv2D(NEURON_BASIS * 2, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS * 2, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS * 2, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = MaxPooling2D(pool_size=(2, 2))(layer)

    layer = Conv2D(NEURON_BASIS * 4, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS * 4, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS * 4, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = MaxPooling2D(pool_size=(2, 2))(layer)

    layer = Conv2D(NEURON_BASIS * 8, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS * 8, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS * 8, (3, 3), activation=ACTIVATION, padding='same')(layer)
    layer = Conv2D(NEURON_BASIS * 4, (1, 1), activation=ACTIVATION, padding='same')(layer)
    layer = MaxPooling2D(pool_size=(2, 2))(layer)

    layer = Flatten()(layer)
    layer = BatchNormalization()(layer)
    layer = Dense(512, activation=ACTIVATION)(layer)
    layer = Dropout(DROPOUT)(layer)
    layer = Dense(2048, activation=ACTIVATION)(layer)
    layer = Dropout(DROPOUT)(layer)
    layer = Dense(tag_number, activation='sigmoid')(layer)

    model = Model(inputs=[inputs], outputs=[layer])
    sgd = optimizers.SGD(lr=LEARNING_RATE, momentum=MOMENTUM, nesterov=True)
    model.compile(optimizer='adam', loss='binary_crossentropy')
    model.summary()

    return model

def get_tags(root, files, min_tag_use=MIN_TAG_USE, suffix=TAG_END):
    with open(join('..', 'imgs', root + '.json'), 'r') as file:
        tags = json.load(file)
    tag_count = {}
    files = [os.path.split(file)[-1] for file in files]
    for key, value in tags.items():
        if key + f'{FILE_END}.png' not in files:
            continue
        for tag in value.split():
            if tag not in tag_count:
                tag_count[tag] = 1
            else:
                tag_count[tag] += 1
    with open(join('..', 'imgs', root + '_count.json'), 'w') as file:
        json.dump(tag_count, file, sort_keys=True, indent=4)
    print(f'Have {len(list(tag_count.keys()))} tags')
    tags_count = {tag: count for tag, count in tag_count.items() if count > min_tag_use and tag.endswith(suffix)}
    print(f'Keep tags with >{min_tag_use} use: {len(tag_count)} tags')
    for tag, count in tags_count.items():
        print(f'{tag}: {count}')
    input('Continue?')
    return tags, tags_count

def make_output(files, tags, tags_count):
    output = {}
    for file in tqdm(files):
        i = os.path.splitext(os.path.split(file)[-1])[0]
        if FILE_END:
            i = i[:-1]
        truth = tags[i].split()
        output[file] = []
        for tag in tags_count.keys():
            if tag in truth:
                output[file].append(1)
            else:
                output[file].append(0)
    return output

def metrics(model, files, output, tags_count):
    true_positives = np.zeros(len(output))
    positives = np.zeros(len(output))
    truth = np.zeros(len(output))
    for file in tqdm(files):
        img = image_process(file)
        img = np.expand_dims(img, axis=0)
        prediction = model.predict(img)[0]
        for i, coef in enumerate(prediction):
            f = tags_count.values()[i] / len(files)
            if coef > f:
                positives[i] += 1
            if output[file][i] > f:
                truth[i] += 1
            if output[file][i] > f and coef > f:
                true_positives[i] += 1

    print('Tag\tPrecision\tRecall')
    for i, k, l, key in zip(true_positives, positives, truth, tags_count.keys()):
        if k != 0:
            precision = int(i / k * 1000) / 100
        else:
            precision = 0
        if l != 0:
            recall = int(i / l * 1000) / 100
        else:
            recall = 0
        print(f'{key}\t{precision}%\t{recall}%\t')

# @functools.lru_cache(maxsize=IMAGE_NUMBER)
def image_process(file):
    img = cv2.imread(file)
    img = img[:, :, [2, 1, 0]]
    # img = resize(img, INPUT_SHAPE, mode='reflect', preserve_range=True, anti_aliasing=True)
    return img

def generator(files, output, batch_size=BATCH_SIZE):
    while True:
        batch_files = np.random.choice(files, size=batch_size)
        # j += 1
        # print(index, j, [(k + j) % n for k in index], [(k + j) for k in index], index+j)
        batch_output = np.array([output[file] for file in batch_files])

        batch_input = np.zeros([batch_size] + [shape for shape in INPUT_SHAPE])
        for i, file in enumerate(batch_files):
            batch_input[i] = image_process(file)

        yield batch_input / 255, batch_output

def train(model, files, output, tags_count, weight_filename=WEIGHT_FILENAME,
          validation_split=VALIDATION_SPLIT, epochs=EPOCHS, batch_size=BATCH_SIZE):
    class_weights = {i: len(files) / count for i, count in enumerate(tags_count.values())}
    index = int(len(files) * validation_split)
    training_generator = generator(files[:index], output)
    validation_generator = generator(files[index:], output)

    calls = [ModelCheckpoint(weight_filename, save_best_only=True),
             PlotLearning(examples=True)]
    model.example = next(validation_generator)
    model.labels = list(tags_count.keys())

    model.fit_generator(generator=training_generator,
                        validation_data=validation_generator,
                        verbose=1,
                        steps_per_epoch=int(len(files) * validation_split) // batch_size,
                        validation_steps=int(len(files) * (1 - validation_split)) // batch_size,
                        epochs=epochs,
                        callbacks=calls,
                        class_weight=class_weights
                        )

def test(files, output, tags_count, weight_filename=WEIGHT_FILENAME):
    model = load_model(weight_filename)
    metrics(model, files, output, tags_count)
    image_generator = generator(files, output, batch_size=1)
    fs = [count / len(files) for count in tags_count.values()]
    fs = [0.5 for i in fs]
    while True:
        print('---')
        im, truth = next(image_generator)
        truth_string = ' '.join([tags_count.keys()[j] for j, v in enumerate(truth[0]) if v > fs[j]])
        print('TRUTH:', truth_string)
        print(im.shape)

        prediction = model.predict(im)[0]
        prediction_string = ' '.join([tags_count.keys()[j] for j, v in enumerate(prediction) if v > fs[j]])
        print('PREDICTION:', prediction_string)
        plt.imshow(im[0])
        plt.show()

def main():
    root = join('..', 'imgs', ROOT)
    files = [join(root, folder, file) for folder in os.listdir(root) for file in os.listdir(join(root, folder))][
            :IMAGE_NUMBER]
    tags, tags_count = get_tags(ROOT, files)
    output = make_output(files, tags, tags_count)
    # test(files, output, tags_count)
    model = import_model(len(tags_count))
    train(model, files, output, tags_count)
    print('DONE')

if __name__ == '__main__':
    main()


# Author: Lars Buitinck
# License: 3-clause BSD

import numpy as np
from ..base import BaseEstimator
from .base import SelectorMixin
from ..utils import check_array
from ..utils.sparsefuncs import mean_variance_axis
from ..utils.validation import check_is_fitted

class VarianceThreshold(BaseEstimator, SelectorMixin):
    """Feature selector that removes all low-variance features.

    This feature selection algorithm looks only at the features (X), not the
    desired outputs (y), and can thus be used for unsupervised learning.

    Read more in the :ref:`User Guide <variance_threshold>`.

    Parameters
    ----------
    threshold : float, optional
        Features with a training-set variance lower than this threshold will
        be removed. The default is to keep all features with non-zero variance,
        i.e. remove the features that have the same value in all samples.

    Attributes
    ----------
    variances_ : array, shape (n_features,)
        Variances of individual features.

    Examples
    --------
    The following dataset has integer features, two of which are the same
    in every sample. These are removed with the default setting for threshold::

        >>> X = [[0, 2, 0, 3], [0, 1, 4, 3], [0, 1, 1, 3]]
        >>> selector = VarianceThreshold()
        >>> selector.fit_transform(X)
        array([[2, 0],
               [1, 4],
               [1, 1]])
    """

    def __init__(self, threshold=0.):
        self.threshold = threshold

    def fit(self, X, y=None):
        """Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self
        """
        X = check_array(X, ('csr', 'csc'), dtype=np.float64)

        if hasattr(X, "toarray"):   # sparse matrix
            _, self.variances_ = mean_variance_axis(X, axis=0)
        else:
            self.variances_ = np.var(X, axis=0)

        if np.all(self.variances_ <= self.threshold):
            msg = "No feature in X meets the variance threshold {0:.5f}"
            if X.shape[0] == 1:
                msg += " (X contains only one sample)"
            raise ValueError(msg.format(self.threshold))

        return self

    def _get_support_mask(self):
        check_is_fitted(self, 'variances_')

        return self.variances_ > self.threshold