"""Test truncated SVD transformer."""

import numpy as np
import scipy.sparse as sp

from sklearn.decomposition import TruncatedSVD
from sklearn.utils import check_random_state
from sklearn.utils.testing import (assert_array_almost_equal, assert_equal,
                                   assert_raises, assert_greater,
                                   assert_array_less)

# Make an X that looks somewhat like a small tf-idf matrix.
# XXX newer versions of SciPy have scipy.sparse.rand for this.
shape = 60, 55
n_samples, n_features = shape
rng = check_random_state(42)
X = rng.randint(-100, 20, np.product(shape)).reshape(shape)
X = sp.csr_matrix(np.maximum(X, 0), dtype=np.float64)
X.data[:] = 1 + np.log(X.data)
Xdense = X.A

def test_algorithms():
    svd_a = TruncatedSVD(30, algorithm="arpack")
    svd_r = TruncatedSVD(30, algorithm="randomized", random_state=42)

    Xa = svd_a.fit_transform(X)[:, :6]
    Xr = svd_r.fit_transform(X)[:, :6]
    assert_array_almost_equal(Xa, Xr)

    comp_a = np.abs(svd_a.components_)
    comp_r = np.abs(svd_r.components_)
    # All elements are equal, but some elements are more equal than others.
    assert_array_almost_equal(comp_a[:9], comp_r[:9])
    assert_array_almost_equal(comp_a[9:], comp_r[9:], decimal=3)

def test_attributes():
    for n_components in (10, 25, 41):
        tsvd = TruncatedSVD(n_components).fit(X)
        assert_equal(tsvd.n_components, n_components)
        assert_equal(tsvd.components_.shape, (n_components, n_features))

def test_too_many_components():
    for algorithm in ["arpack", "randomized"]:
        for n_components in (n_features, n_features+1):
            tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm)
            assert_raises(ValueError, tsvd.fit, X)

def test_sparse_formats():
    for fmt in ("array", "csr", "csc", "coo", "lil"):
        Xfmt = Xdense if fmt == "dense" else getattr(X, "to" + fmt)()
        tsvd = TruncatedSVD(n_components=11)
        Xtrans = tsvd.fit_transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))
        Xtrans = tsvd.transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))

def test_inverse_transform():
    for algo in ("arpack", "randomized"):
        # We need a lot of components for the reconstruction to be "almost
        # equal" in all positions. XXX Test means or sums instead?
        tsvd = TruncatedSVD(n_components=52, random_state=42)
        Xt = tsvd.fit_transform(X)
        Xinv = tsvd.inverse_transform(Xt)
        assert_array_almost_equal(Xinv, Xdense, decimal=1)

def test_integers():
    Xint = X.astype(np.int64)
    tsvd = TruncatedSVD(n_components=6)
    Xtrans = tsvd.fit_transform(Xint)
    assert_equal(Xtrans.shape, (n_samples, tsvd.n_components))

def test_explained_variance():
    # Test sparse data
    svd_a_10_sp = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_sp = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_sp = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_sp = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_sp = svd_a_10_sp.fit_transform(X)
    X_trans_r_10_sp = svd_r_10_sp.fit_transform(X)
    X_trans_a_20_sp = svd_a_20_sp.fit_transform(X)
    X_trans_r_20_sp = svd_r_20_sp.fit_transform(X)

    # Test dense data
    svd_a_10_de = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_de = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_de = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_de = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_de = svd_a_10_de.fit_transform(X.toarray())
    X_trans_r_10_de = svd_r_10_de.fit_transform(X.toarray())
    X_trans_a_20_de = svd_a_20_de.fit_transform(X.toarray())
    X_trans_r_20_de = svd_r_20_de.fit_transform(X.toarray())

    # helper arrays for tests below
    svds = (svd_a_10_sp, svd_r_10_sp, svd_a_20_sp, svd_r_20_sp, svd_a_10_de,
            svd_r_10_de, svd_a_20_de, svd_r_20_de)
    svds_trans = (
        (svd_a_10_sp, X_trans_a_10_sp),
        (svd_r_10_sp, X_trans_r_10_sp),
        (svd_a_20_sp, X_trans_a_20_sp),
        (svd_r_20_sp, X_trans_r_20_sp),
        (svd_a_10_de, X_trans_a_10_de),
        (svd_r_10_de, X_trans_r_10_de),
        (svd_a_20_de, X_trans_a_20_de),
        (svd_r_20_de, X_trans_r_20_de),
    )
    svds_10_v_20 = (
        (svd_a_10_sp, svd_a_20_sp),
        (svd_r_10_sp, svd_r_20_sp),
        (svd_a_10_de, svd_a_20_de),
        (svd_r_10_de, svd_r_20_de),
    )
    svds_sparse_v_dense = (
        (svd_a_10_sp, svd_a_10_de),
        (svd_a_20_sp, svd_a_20_de),
        (svd_r_10_sp, svd_r_10_de),
        (svd_r_20_sp, svd_r_20_de),
    )

    # Assert the 1st component is equal
    for svd_10, svd_20 in svds_10_v_20:
        assert_array_almost_equal(
            svd_10.explained_variance_ratio_,
            svd_20.explained_variance_ratio_[:10],
            decimal=5,
        )

    # Assert that 20 components has higher explained variance than 10
    for svd_10, svd_20 in svds_10_v_20:
        assert_greater(
            svd_20.explained_variance_ratio_.sum(),
            svd_10.explained_variance_ratio_.sum(),
        )

    # Assert that all the values are greater than 0
    for svd in svds:
        assert_array_less(0.0, svd.explained_variance_ratio_)

    # Assert that total explained variance is less than 1
    for svd in svds:
        assert_array_less(svd.explained_variance_ratio_.sum(), 1.0)

    # Compare sparse vs. dense
    for svd_sparse, svd_dense in svds_sparse_v_dense:
        assert_array_almost_equal(svd_sparse.explained_variance_ratio_,
                svd_dense.explained_variance_ratio_)

    # Test that explained_variance is correct
    for svd, transformed in svds_trans:
        total_variance = np.var(X.toarray(), axis=0).sum()
        variances = np.var(transformed, axis=0)
        true_explained_variance_ratio = variances / total_variance

        assert_array_almost_equal(
            svd.explained_variance_ratio_,
            true_explained_variance_ratio,
        )


"""@file deepclusteringnoise_reconstructor.py
contains the reconstor class using deep clustering for modified noise architecture"""

from sklearn.cluster import KMeans
import mask_reconstructor
from nabu.postprocessing import data_reader
import numpy as np
import os

class DeepclusteringnoiseReconstructor(mask_reconstructor.MaskReconstructor):
	"""the deepclusteringnoise reconstructor class for modified architecture for noise

	a reconstructor using deep clustering"""

	requested_output_names = ['bin_emb', 'noise_filter']

	def __init__(self, conf, evalconf, dataconf, rec_dir, task, optimal_frame_permutation=False):
		"""DeepclusteringnoiseReconstructor constructor

		Args:
			conf: the reconstructor configuration as a dictionary
			evalconf: the evaluator configuration as a ConfigParser
			dataconf: the database configuration
			rec_dir: the directory where the reconstructions will be stored
			task: name of task
		"""

		super(DeepclusteringnoiseReconstructor, self).__init__(
			conf, evalconf, dataconf, rec_dir, task, optimal_frame_permutation)

		if 'noise_threshold_for_kmeans' in conf:
			self.noise_threshold = float(conf['noise_threshold_for_kmeans'])
		else:
			self.noise_threshold = 0.75
		if 'min_kmeans_perc' in conf:
			self.min_kmeans_perc = float(conf['min_kmeans_perc'])
		else:
			self.min_kmeans_perc = 0.05

		# get the usedbins reader
		usedbins_names = conf['usedbins'].split(' ')
		usedbins_dataconfs = []
		for usedbins_name in usedbins_names:
			usedbins_dataconfs.append(dict(dataconf.items(usedbins_name)))
		self.usedbins_reader = data_reader.DataReader(usedbins_dataconfs, self.segment_lengths)

		# directory where cluster centroids will be stored
		self.center_store_dir = os.path.join(rec_dir, 'cluster_centers')
		if not os.path.isdir(self.center_store_dir):
			os.makedirs(self.center_store_dir)

	def _get_masks(self, output, utt_info):
		"""estimate the masks

		Args:
			output: the output of a single utterance of the neural network
				utt_info: some info on the utterance

		Returns:
			the estimated masks"""

		embeddings = output['bin_emb']  # Embeddingvectors
		noise_filter = output['noise_filter']  # noise filter output network (alpha)
		# only the non-silence bins will be used for the clustering
		usedbins, _ = self.usedbins_reader(self.pos)

		[T, F] = np.shape(usedbins)
		emb_dim = np.shape(embeddings)[1]/F

		if np.shape(embeddings)[0] != T:
			raise Exception('Number of frames in usedbins does not match the sequence length')
		if np.shape(noise_filter)[0] != T:
			raise Exception('Number of frames in noise detect does not match the sequence length')
		if np.shape(noise_filter)[1] != F:
			raise Exception('Number of noise detect outputs does not match number of frequency bins')

		# reshape the embeddings vectors
		embeddings = embeddings[:T, :]
		embeddings_resh = np.reshape(embeddings, [T*F, emb_dim])
		embeddings_resh_norm = np.linalg.norm(embeddings_resh, axis=1, keepdims=True)
		embeddings_resh = embeddings_resh/embeddings_resh_norm

		if np.isnan(embeddings_resh).any():
			print 'Embedding reshape contains NaN'

		# reshape noise filter
		noise_filter = noise_filter[:T, :]
		noise_filter_resh = np.reshape(noise_filter, T*F)
		# which cells have not too much noise
		no_noise = noise_filter_resh > self.noise_threshold
		# only keep the active bins (above threshold) for clustering and not too noisy
		usedbins_resh = np.reshape(usedbins, T*F)
		filt = np.logical_and(usedbins_resh, no_noise)
		perc_for_kmeans = float(np.sum(filt))/float(np.sum(usedbins_resh))
		if perc_for_kmeans < self.min_kmeans_perc:
			print \
				'Found that less then %.1f%% (%.1f%%)of the tf bins with energy where considered non-noise for the Kmeans. ' \
				'Lowering the noise threshold so that %.1f%% of the bins will be considered' % \
				(self.min_kmeans_perc*100, perc_for_kmeans*100, self.min_kmeans_perc*100)
			num_bins_wanted = int(np.ceil(np.sum(usedbins_resh)*self.min_kmeans_perc))
			noise_filt_used_bin = noise_filter_resh * usedbins_resh
			sorted_noise_filt_used_bin_inds = np.argsort(noise_filt_used_bin)
			sorted_noise_filt_used_bin_inds = sorted_noise_filt_used_bin_inds[::-1]
			filt = sorted_noise_filt_used_bin_inds[:num_bins_wanted]

		embeddings_speech_resh = embeddings_resh[filt]
		if np.isnan(embeddings_speech_resh).any():
			print 'embeddings_speech_resh contains NaN'
		if np.shape(embeddings_speech_resh)[0] < 2:
			return np.zeros([self.nrS, T, F])
		# apply kmeans clustering and assign each bin to a cluster
		kmeans_model = KMeans(n_clusters=self.nrS, init='k-means++', n_init=10, max_iter=100, n_jobs=-1)

		# Sometime it fails due to some indexerror and I'm not sure why. Just retry then. max 5 times
		for _ in range(5):
			try:
				kmeans_model.fit(embeddings_speech_resh)
			except IndexError:
				continue
			break
		# assign each cell to cluster
		predicted_labels = kmeans_model.predict(embeddings_resh)
		predicted_labels_resh = np.reshape(predicted_labels, [T, F])

		# reconstruct the masks from the cluster labels
		masks = np.zeros([self.nrS, T, F])
		for spk in range(self.nrS):
			masks[spk, :, :] = (predicted_labels_resh == spk)*noise_filter
		if np.isnan(masks).any():
			print 'masks contains NaN'

		# store the clusters
		np.save(os.path.join(self.center_store_dir, utt_info['utt_name']), kmeans_model.cluster_centers_)

		return masks


import json
import shapefile as shp
import matplotlib.pyplot as plt
import random

def mean(numbers):
    return float(sum(numbers)) / max(len(numbers), 1)

numbersX = []
numbersY = []

TaskPoints = {}

shpFilePath = r"D:\TaxiCab\mycode\Plots\ShapefileAndTrajectory\taxi_zones\taxi_zones"

sf = shp.Reader(shpFilePath)
records = sf.records()

plt.figure()
for shape in sf.shapeRecords():
    #print(records[0][3])
    x = [i[0] for i in shape.shape.points[:]]
    meanX = mean(x)
    numbersX.append(meanX)
    y = [i[1] for i in shape.shape.points[:]]
    meanY = mean(y)
    numbersY.append(meanY)
    plt.plot(x,y)

num = 0 ##range(263)
for x, y in zip(numbersX, numbersY):
    plt.text(x, y, str(num), color="red", fontsize=12)
    num = num + 1

plt.plot(numbersX, numbersY, 'o', color='blue', markersize=7, markeredgewidth=0.0)
#print (len(numbersX))
#print (numbersY)
plt.show()

Date_min = 1
Date_max = 30
Beta_min = 2
Beta_max = 30
#print (range(len(numbersX)))
for i in range(len(numbersX)):
    date = "2017/1/"
    TaskPoints_trace = []
    TaskPoints_trace.append(records[i][3])
    TaskPoints_trace.append(numbersX[i])
    TaskPoints_trace.append(numbersY[i])
    TaskPoints_trace.append(random.randint(Beta_min, Beta_max))
    date += str(random.randint(Date_min, Date_max))
    TaskPoints_trace.append(date)
    TaskPoints[i] = TaskPoints_trace

json.dump(TaskPoints, open('Data1/TaxiZone_TaskPoints.json', 'w'), indent=4, sort_keys=True, separators=(',', ':'))


import inspect
import gc
import sys
import os.path
import difflib
from collections import OrderedDict

import pandas as pd
from pandas.core.common import in_ipnb

def is_property(code):
    """
    Using some CPython gc magics, check if a code object is a property

    gc idea taken from trace.py from stdlib
    """
    ## use of gc.get_referrers() was suggested by Michael Hudson
    # all functions which refer to this code object
    gc.collect()
    code_refs = gc.get_referrers(code)
    funcs = [f for f in code_refs
                    if inspect.isfunction(f)]
    if len(funcs) != 1:
        return False

    # property object will reference the original func
    props = [p for p in gc.get_referrers(funcs[0])
                    if isinstance(p, property)]
    return len(props) == 1

def is_class_dict(dct):
    if not isinstance(dct, dict):
        return False
    if '__dict__' not in dct or not inspect.isgetsetdescriptor(dct['__dict__']):
        return False
    return True

def get_parent(code):
    funcs = [f for f in gc.get_referrers(code)
                    if inspect.isfunction(f)]

    if len(funcs) != 1:
        return None

    refs = [f for f in gc.get_referrers(funcs[0])]

    for ref in refs:
        # assume if that if a dict is pointed to by a class,
        # that dict is the __dict__
        if isinstance(ref, dict):
            parents = [p for p in gc.get_referrers(ref) if isinstance(p, type)]
            if len(parents) == 1:
                return parents[0].__name__

        if inspect.ismethod(ref):
            return ref.__qualname__.rsplit('.', 1)[0]

    return None

class Follow(object):
    """
    Follows execution path.

    Meant as a quick way to see what a function does.

    In [2]: with Follow() as f:
    ...:     df.sum()
    ...:

    In [3]: f.pprint(depth=2)
    stat_func generic.py:3542
        _reduce frame.py:3995
            _get_axis_number generic.py:285
            _get_agg_axis frame.py:4128
            as_matrix generic.py:1938
    """
    def __init__(self, depth=1, silent=False, parent=False):
        self.depth = depth
        self.silent = silent
        self.timings = []
        self.frame_cache = {}
        self._caller_cache = {}
        self.parent = parent

        self.stack_depth_cache = {}

    def current_depth(self, frame):
        current_depth = None
        i = 0
        f = frame.f_back
        while f:
            i += 1
            parent_depth = self.stack_depth_cache.get(id(f), None)
            if parent_depth is not None:
                current_depth = i + parent_depth
                break
            # if we're already past depth, don't bother finding real depth
            if i > self.depth:
                return None
            f = f.f_back

        # should always at least get back to base parent
        return current_depth

    def trace_dispatch(self, frame, event, arg):
        if len(self.stack_depth_cache) == 0:
            # __enter__ is the intial frame
            self.stack_depth_cache[id(frame.f_back)] = 0

        # the lower parts get heavy. don't do anything or frames deeper
        # than depth
        current_depth = self.current_depth(frame)
        if current_depth is None:
            return

        if current_depth > self.depth:
            return

        if event not in ['call', 'c_call']:
            return

        # skip built in funcs
        if inspect.isbuiltin(arg):
            return

        # skip properties, we're only really interested in function calls
        # this will unfortunently skip any important logic that is wrapped
        # in property logic
        code = frame.f_code
        if is_property(code):
            return

        # note that get_parent is supa slow.
        parent_name = None
        if self.parent:
            parent_name = get_parent(code)

        indent, first_parent = self.indent_level(frame)
        f = frame.f_back
        if event == "c_call":
            func_name = arg.__name__
            fn = (indent, "", 0, func_name, id(frame),id(first_parent), None)
        elif event == 'call':
            fcode = frame.f_code
            fn = (indent, fcode.co_filename, fcode.co_firstlineno,
                  fcode.co_name, id(frame), id(first_parent), parent_name)

        self.timings.append(fn)

    def indent_level(self, frame):
        i = 0
        f = frame.f_back
        first_parent = f
        while f:
            if id(f) in self.frame_cache:
                i += 1
            f = f.f_back
        if i == 0:
            # clear out the frame cache
            self.frame_cache = {id(frame): True}
        else:
            self.frame_cache[id(frame)] = True
        return i, first_parent

    def to_frame(self):
        data = self.timings
        cols = ['indent', 'filename', 'lineno', 'func_name', 'frame_id',
                'parent_id', 'parent_name']
        df = pd.DataFrame(data, columns=cols)
        df.loc[:, 'filename'] = df.filename.apply(lambda s: os.path.basename(s))
        return df

    def __enter__(self):
        sys.setprofile(self.trace_dispatch)
        return self

    def __exit__(self, type, value, traceback):
        sys.setprofile(None)
        if not self.silent:
            self.pprint(self.depth)

    def file_module_function_of(self, frame):
        code = frame.f_code
        filename = code.co_filename
        if filename:
            modulename = modname(filename)
        else:
            modulename = None

        funcname = code.co_name
        clsname = None
        if code in self._caller_cache:
            if self._caller_cache[code] is not None:
                clsname = self._caller_cache[code]
        else:
            self._caller_cache[code] = None
            ## use of gc.get_referrers() was suggested by Michael Hudson
            # all functions which refer to this code object
            funcs = [f for f in gc.get_referrers(code)
                         if inspect.isfunction(f)]
            # require len(func) == 1 to avoid ambiguity caused by calls to
            # new.function(): "In the face of ambiguity, refuse the
            # temptation to guess."
            if len(funcs) == 1:
                dicts = [d for d in gc.get_referrers(funcs[0])
                             if isinstance(d, dict)]
                if len(dicts) == 1:
                    classes = [c for c in gc.get_referrers(dicts[0])
                                   if hasattr(c, "__bases__")]
                    if len(classes) == 1:
                        # ditto for new.classobj()
                        clsname = classes[0].__name__
                        # cache the result - assumption is that new.* is
                        # not called later to disturb this relationship
                        # _caller_cache could be flushed if functions in
                        # the new module get called.
                        self._caller_cache[code] = clsname
        if clsname is not None:
            funcname = "%s.%s" % (clsname, funcname)

        return filename, modulename, funcname

    def gen_output(self, depth=None):
        df = self.to_frame()
        mask = df.filename == ''
        mask = mask | df.func_name.isin(['<lambda>', '<genexpr>'])
        mask = mask | df.func_name.str.startswith('__')
        if depth:
            mask = mask | (df.indent > depth)

        MSG_FORMAT = "{indent}{func_name}{class_name} <{filename}:{lineno}>"

        df = df.loc[~mask]
        def format(row):
            indent = row[0]
            filename = row[1]
            lineno = row[2]
            func_name = row[3]
            class_name = row[6] or ''
            if class_name:
                class_name = '::{class_name}'.format(class_name=class_name)

            msg = MSG_FORMAT.format(indent=" "*indent*4, func_name=func_name,
                                    filename=filename, lineno=lineno,
                                    class_name=class_name)
            return msg

        df = df.reset_index(drop=True)

        output = df.apply(format, axis=1, raw=True)

        return output.tolist()

    def pprint(self, depth=None):
        output = self.gen_output(depth=depth)
        print(("-" * 40))
        print(("Follow Path (depth {depth}):".format(depth=depth)))
        print(("-" * 40))
        print(("\n".join(output)))

    def diff(self, right, depth):
        if in_ipnb():
            return self._html_diff(right=right, depth=depth)
        else:
            return self._text_diff(right=right, depth=depth)

    def _text_diff(self, right, depth):
        output = self.gen_output(depth)
        output2 = right.gen_output(depth)

        htmldiff = difflib.HtmlDiff()
        return '\n'.join(difflib.ndiff(output, output2))

    def _html_diff(self, right, depth):
        from IPython.core.display import HTML
        output = self.gen_output(depth)
        output2 = right.gen_output(depth)

        htmldiff = difflib.HtmlDiff()
        diff = htmldiff.make_table(output, output2)
        return HTML(diff)


# -*- coding: UTF-8 -*-
"""
Plot visualization of user habits, i.e. show which actions typically follow some given user action.

Note: the figure for "Frontdoor=Closed" slightly deviates from Figure 1 in the paper and Figure 5.1 in the
dissertation (see paper_experiments.py for bibliographical information). The number of observed actions was reported
correctly in the paper/dissertation, however there was an issue with ordering which actions occur most commonly,
which resulted in "Open cups cupboard" being erroneously included in the figure. Despite this issue, the main point
of the figure still stands: the user has some observable habits after closing the frontdoor.
"""

import sys
sys.path.append("..")

import pandas

from recsys.classifiers.temporal import TemporalEvidencesClassifier
from recsys.classifiers.binning import initialize_bins
from recsys.dataset import load_dataset
from evaluation import plot
import config

#configuration
data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config")

#fit classifier to dataset
cls = TemporalEvidencesClassifier(data.features, data.target_names, bins=initialize_bins(0, 300, 10))
cls = cls.fit(data.data, data.target)

#create visualizations of habits around each user action
plot_conf = plot.plot_config(config.plot_directory, sub_dirs=[data.name, "habits"], img_type=config.img_type)
for source in cls.sources.values():
    observations = pandas.DataFrame(source.temporal_counts)
    observations.columns = data.target_names
    observations.index = cls.bins
    plot.plot_observations(source.name(), observations, plot_conf)

print "Results can be found in the \"%s\" directory" % config.plot_directory


import os
import shutil
import tempfile
import warnings
import nose
import numpy
from pickle import loads
from pickle import dumps

from sklearn.datasets import get_data_home
from sklearn.datasets import clear_data_home
from sklearn.datasets import load_files
from sklearn.datasets import load_sample_images
from sklearn.datasets import load_sample_image
from sklearn.datasets import load_digits
from sklearn.datasets import load_diabetes
from sklearn.datasets import load_linnerud
from sklearn.datasets import load_iris
from sklearn.datasets import load_breast_cancer
from sklearn.datasets import load_boston
from sklearn.datasets.base import Bunch

from sklearn.externals.six import b, u

from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises

DATA_HOME = tempfile.mkdtemp(prefix="scikit_learn_data_home_test_")
LOAD_FILES_ROOT = tempfile.mkdtemp(prefix="scikit_learn_load_files_test_")
TEST_CATEGORY_DIR1 = ""
TEST_CATEGORY_DIR2 = ""

def _remove_dir(path):
    if os.path.isdir(path):
        shutil.rmtree(path)

def teardown_module():
    """Test fixture (clean up) run once after all tests of this module"""
    for path in [DATA_HOME, LOAD_FILES_ROOT]:
        _remove_dir(path)

def setup_load_files():
    global TEST_CATEGORY_DIR1
    global TEST_CATEGORY_DIR2
    TEST_CATEGORY_DIR1 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    TEST_CATEGORY_DIR2 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    sample_file = tempfile.NamedTemporaryFile(dir=TEST_CATEGORY_DIR1,
                                              delete=False)
    sample_file.write(b("Hello World!\n"))
    sample_file.close()

def teardown_load_files():
    _remove_dir(TEST_CATEGORY_DIR1)
    _remove_dir(TEST_CATEGORY_DIR2)

def test_data_home():
    # get_data_home will point to a pre-existing folder
    data_home = get_data_home(data_home=DATA_HOME)
    assert_equal(data_home, DATA_HOME)
    assert_true(os.path.exists(data_home))

    # clear_data_home will delete both the content and the folder it-self
    clear_data_home(data_home=data_home)
    assert_false(os.path.exists(data_home))

    # if the folder is missing it will be created again
    data_home = get_data_home(data_home=DATA_HOME)
    assert_true(os.path.exists(data_home))

def test_default_empty_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 0)
    assert_equal(len(res.target_names), 0)
    assert_equal(res.DESCR, None)

@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_default_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.data, [b("Hello World!\n")])

@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_load_files_w_categories_desc_and_encoding():
    category = os.path.abspath(TEST_CATEGORY_DIR1).split('/').pop()
    res = load_files(LOAD_FILES_ROOT, description="test",
                     categories=category, encoding="utf-8")
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 1)
    assert_equal(res.DESCR, "test")
    assert_equal(res.data, [u("Hello World!\n")])

@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_load_files_wo_load_content():
    res = load_files(LOAD_FILES_ROOT, load_content=False)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.get('data'), None)

def test_load_sample_images():
    try:
        res = load_sample_images()
        assert_equal(len(res.images), 2)
        assert_equal(len(res.filenames), 2)
        assert_true(res.DESCR)
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_digits():
    digits = load_digits()
    assert_equal(digits.data.shape, (1797, 64))
    assert_equal(numpy.unique(digits.target).size, 10)

def test_load_digits_n_class_lt_10():
    digits = load_digits(9)
    assert_equal(digits.data.shape, (1617, 64))
    assert_equal(numpy.unique(digits.target).size, 9)

def test_load_sample_image():
    try:
        china = load_sample_image('china.jpg')
        assert_equal(china.dtype, 'uint8')
        assert_equal(china.shape, (427, 640, 3))
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_missing_sample_image_error():
    have_PIL = True
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
    except ImportError:
        have_PIL = False
    if have_PIL:
        assert_raises(AttributeError, load_sample_image,
                      'blop.jpg')
    else:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_diabetes():
    res = load_diabetes()
    assert_equal(res.data.shape, (442, 10))
    assert_true(res.target.size, 442)

def test_load_linnerud():
    res = load_linnerud()
    assert_equal(res.data.shape, (20, 3))
    assert_equal(res.target.shape, (20, 3))
    assert_equal(len(res.target_names), 3)
    assert_true(res.DESCR)

def test_load_iris():
    res = load_iris()
    assert_equal(res.data.shape, (150, 4))
    assert_equal(res.target.size, 150)
    assert_equal(res.target_names.size, 3)
    assert_true(res.DESCR)

def test_load_breast_cancer():
    res = load_breast_cancer()
    assert_equal(res.data.shape, (569, 30))
    assert_equal(res.target.size, 569)
    assert_equal(res.target_names.size, 2)
    assert_true(res.DESCR)

def test_load_boston():
    res = load_boston()
    assert_equal(res.data.shape, (506, 13))
    assert_equal(res.target.size, 506)
    assert_equal(res.feature_names.size, 13)
    assert_true(res.DESCR)

def test_loads_dumps_bunch():
    bunch = Bunch(x="x")
    bunch_from_pkl = loads(dumps(bunch))
    bunch_from_pkl.x = "y"
    assert_equal(bunch_from_pkl['x'], bunch_from_pkl.x)

def test_bunch_pickle_generated_with_0_16_and_read_with_0_17():
    bunch = Bunch(key='original')
    # This reproduces a problem when Bunch pickles have been created
    # with scikit-learn 0.16 and are read with 0.17. Basically there
    # is a suprising behaviour because reading bunch.key uses
    # bunch.__dict__ (which is non empty for 0.16 Bunch objects)
    # whereas assigning into bunch.key uses bunch.__setattr__. See
    # https://github.com/scikit-learn/scikit-learn/issues/6196 for
    # more details
    bunch.__dict__['key'] = 'set from __dict__'
    bunch_from_pkl = loads(dumps(bunch))
    # After loading from pickle the __dict__ should have been ignored
    assert_equal(bunch_from_pkl.key, 'original')
    assert_equal(bunch_from_pkl['key'], 'original')
    # Making sure that changing the attr does change the value
    # associated with __getitem__ as well
    bunch_from_pkl.key = 'changed'
    assert_equal(bunch_from_pkl.key, 'changed')
    assert_equal(bunch_from_pkl['key'], 'changed')


from benchmark_functions import *
from pso import *

import matplotlib.pyplot as plt

iterations = 100
particles = 500
dimensions = 2
search_space = [[-5.12] * dimensions, [5.12] * dimensions]
# print init_pso(iterations, particles, search_space)
velocity, fitness, local_best, local_position, global_best, global_position = init_pso(iterations, particles,
                                                                                search_space)
# print create_swarm(particles, search_space)
swarm = create_swarm(particles, search_space)

iteration = 0
while iteration < iterations:
    fitness = [sphere(solution) for solution in swarm]

    local_best, local_position = update_local_position(swarm, fitness, local_best, local_position)

    global_best, global_position = update_global_position(swarm, local_best, global_best, global_position, iteration)

    swarm, velocity = update_swarm(swarm, velocity, local_position, global_position, iteration)

    swarm = check_swarm(swarm, search_space)

    iteration += 1

plt.plot(global_best, '.-', label='%f' % min(global_best))
plt.xlim(-1, iteration)
# plt.ylim(min(global_best), max(global_best)+0.01)
plt.legend()
plt.show()


from __future__ import division

from pandas import Interval

import pytest
import pandas.util.testing as tm

@pytest.fixture
def interval():
    return Interval(0, 1)

class TestInterval(object):

    def test_properties(self, interval):
        assert interval.closed == 'right'
        assert interval.left == 0
        assert interval.right == 1
        assert interval.mid == 0.5

    def test_repr(self, interval):
        assert repr(interval) == "Interval(0, 1, closed='right')"
        assert str(interval) == "(0, 1]"

        interval_left = Interval(0, 1, closed='left')
        assert repr(interval_left) == "Interval(0, 1, closed='left')"
        assert str(interval_left) == "[0, 1)"

    def test_contains(self, interval):
        assert 0.5 in interval
        assert 1 in interval
        assert 0 not in interval

        msg = "__contains__ not defined for two intervals"
        with tm.assert_raises_regex(TypeError, msg):
            interval in interval

        interval_both = Interval(0, 1, closed='both')
        assert 0 in interval_both
        assert 1 in interval_both

        interval_neither = Interval(0, 1, closed='neither')
        assert 0 not in interval_neither
        assert 0.5 in interval_neither
        assert 1 not in interval_neither

    def test_equal(self):
        assert Interval(0, 1) == Interval(0, 1, closed='right')
        assert Interval(0, 1) != Interval(0, 1, closed='left')
        assert Interval(0, 1) != 0

    def test_comparison(self):
        with tm.assert_raises_regex(TypeError, 'unorderable types'):
            Interval(0, 1) < 2

        assert Interval(0, 1) < Interval(1, 2)
        assert Interval(0, 1) < Interval(0, 2)
        assert Interval(0, 1) < Interval(0.5, 1.5)
        assert Interval(0, 1) <= Interval(0, 1)
        assert Interval(0, 1) > Interval(-1, 2)
        assert Interval(0, 1) >= Interval(0, 1)

    def test_hash(self, interval):
        # should not raise
        hash(interval)

    def test_math_add(self, interval):
        expected = Interval(1, 2)
        actual = interval + 1
        assert expected == actual

        expected = Interval(1, 2)
        actual = 1 + interval
        assert expected == actual

        actual = interval
        actual += 1
        assert expected == actual

        msg = "unsupported operand type\(s\) for \+"
        with tm.assert_raises_regex(TypeError, msg):
            interval + Interval(1, 2)

        with tm.assert_raises_regex(TypeError, msg):
            interval + 'foo'

    def test_math_sub(self, interval):
        expected = Interval(-1, 0)
        actual = interval - 1
        assert expected == actual

        actual = interval
        actual -= 1
        assert expected == actual

        msg = "unsupported operand type\(s\) for -"
        with tm.assert_raises_regex(TypeError, msg):
            interval - Interval(1, 2)

        with tm.assert_raises_regex(TypeError, msg):
            interval - 'foo'

    def test_math_mult(self, interval):
        expected = Interval(0, 2)
        actual = interval * 2
        assert expected == actual

        expected = Interval(0, 2)
        actual = 2 * interval
        assert expected == actual

        actual = interval
        actual *= 2
        assert expected == actual

        msg = "unsupported operand type\(s\) for \*"
        with tm.assert_raises_regex(TypeError, msg):
            interval * Interval(1, 2)

        msg = "can\'t multiply sequence by non-int"
        with tm.assert_raises_regex(TypeError, msg):
            interval * 'foo'

    def test_math_div(self, interval):
        expected = Interval(0, 0.5)
        actual = interval / 2.0
        assert expected == actual

        actual = interval
        actual /= 2.0
        assert expected == actual

        msg = "unsupported operand type\(s\) for /"
        with tm.assert_raises_regex(TypeError, msg):
            interval / Interval(1, 2)

        with tm.assert_raises_regex(TypeError, msg):
            interval / 'foo'


from __future__ import division

import numpy as np

from sklearn.utils.linear_assignment_ import linear_assignment
from sklearn.utils.validation import check_consistent_length, check_array

__all__ = ["consensus_score"]

def _check_rows_and_columns(a, b):
    """Unpacks the row and column arrays and checks their shape."""
    check_consistent_length(*a)
    check_consistent_length(*b)
    checks = lambda x: check_array(x, ensure_2d=False)
    a_rows, a_cols = map(checks, a)
    b_rows, b_cols = map(checks, b)
    return a_rows, a_cols, b_rows, b_cols

def _jaccard(a_rows, a_cols, b_rows, b_cols):
    """Jaccard coefficient on the elements of the two biclusters."""
    intersection = ((a_rows * b_rows).sum() *
                    (a_cols * b_cols).sum())

    a_size = a_rows.sum() * a_cols.sum()
    b_size = b_rows.sum() * b_cols.sum()

    return intersection / (a_size + b_size - intersection)

def _pairwise_similarity(a, b, similarity):
    """Computes pairwise similarity matrix.

    result[i, j] is the Jaccard coefficient of a's bicluster i and b's
    bicluster j.

    """
    a_rows, a_cols, b_rows, b_cols = _check_rows_and_columns(a, b)
    n_a = a_rows.shape[0]
    n_b = b_rows.shape[0]
    result = np.array(list(list(similarity(a_rows[i], a_cols[i],
                                           b_rows[j], b_cols[j])
                                for j in range(n_b))
                           for i in range(n_a)))
    return result

def consensus_score(a, b, similarity="jaccard"):
    """The similarity of two sets of biclusters.

    Similarity between individual biclusters is computed. Then the
    best matching between sets is found using the Hungarian algorithm.
    The final score is the sum of similarities divided by the size of
    the larger set.

    Read more in the :ref:`User Guide <biclustering>`.

    Parameters
    ----------
    a : (rows, columns)
        Tuple of row and column indicators for a set of biclusters.

    b : (rows, columns)
        Another set of biclusters like ``a``.

    similarity : string or function, optional, default: "jaccard"
        May be the string "jaccard" to use the Jaccard coefficient, or
        any function that takes four arguments, each of which is a 1d
        indicator vector: (a_rows, a_columns, b_rows, b_columns).

    References
    ----------

    * Hochreiter, Bodenhofer, et. al., 2010. `FABIA: factor analysis
      for bicluster acquisition
      <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2881408/>`__.

    """
    if similarity == "jaccard":
        similarity = _jaccard
    matrix = _pairwise_similarity(a, b, similarity)
    indices = linear_assignment(1. - matrix)
    n_a = len(a[0])
    n_b = len(b[0])
    return matrix[indices[:, 0], indices[:, 1]].sum() / max(n_a, n_b)


import numpy as np
import pandas
import matplotlib.pyplot as plt

def entries_histogram(turnstile_weather):
    '''
    Before we perform any analysis, it might be useful to take a
    look at the data we're hoping to analyze. More specifically, let's
    examine the hourly entries in our NYC subway data and determine what
    distribution the data follows. This data is stored in a dataframe
    called turnstile_weather under the ['ENTRIESn_hourly'] column.

    Let's plot two histograms on the same axes to show hourly
    entries when raining vs. when not raining. Here's an example on how
    to plot histograms with pandas and matplotlib:
    turnstile_weather['column_to_graph'].hist()

    Your histogram may look similar to bar graph in the instructor notes below.

    You can read a bit about using matplotlib and pandas to plot histograms here:
    http://pandas.pydata.org/pandas-docs/stable/visualization.html#histograms

    You can see the information contained within the turnstile weather data here:
    https://www.dropbox.com/s/meyki2wl9xfa7yk/turnstile_data_master_with_weather.csv
    '''

    plt.figure()
    #print turnstile_weather['rain'] == 1
    turnstile_weather[turnstile_weather['rain' ]== 0]['ENTRIESn_hourly'].hist()  # your code here to plot a historgram for hourly entries when it is raining
    turnstile_weather[turnstile_weather['rain'] == 1]['ENTRIESn_hourly'].hist() # your code here to plot a historgram for hourly entries when it is not raining
    return plt


# -*- coding: utf-8 -*-
# <nbformat>3.0</nbformat>

# <codecell>

%matplotlib inline
from pylab import *
import matplotlib.pyplot as plt
from IPython.core.display import Image

# <codecell>

data = []
for y in range(10):
    data.append([y+x for x in range(10)])
# print(data)
Image(data=data)

# <headingcell level=1>

# Matplot Lib

# <codecell>

alpha = 0.7
phi_ext = 2 * pi * 0.5

def flux_qubit_potential(phi_m, phi_p):
    return 2 + alpha - 2 * cos(phi_p)*cos(phi_m) - alpha * cos(phi_ext - 2*phi_p)
phi_m = linspace(0, 2*pi, 100)
phi_p = linspace(0, 2*pi, 100)
X,Y = meshgrid(phi_p, phi_m)
Z = flux_qubit_potential(X, Y).T

# <codecell>

fig, ax = plt.subplots()

p = ax.pcolor(X/(2*pi), Y/(2*pi), Z, cmap=cm.RdBu, vmin=abs(Z).min(), vmax=abs(Z).max())
cb = fig.colorbar(p, ax=ax)

# <codecell>

from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np
import random

bignum = 10
data = []
for i in range(bignum):
    data.append([random.random() for x in range(bignum)])
mat = np.array(data)  #random.random((bignum, bignum))
X, Y = np.mgrid[:bignum, :bignum]

fig = plt.figure()
ax = fig.add_subplot(1,1,1, projection='3d')
surf = ax.plot_surface(X,Y,mat)
plt.show()

# <headingcell level=2>

# Most simple Pixel Map

# <codecell>

def basic_func(x, y):
    return x+y

X, Y = np.mgrid[:bignum, :bignum]
Z = basic_func(X, Y)

# <codecell>

fig, ax = plt.subplots()
p = ax.pcolor(X, Y, Z, cmap=cm.RdBu, vmin=abs(Z).min(), vmax=abs(Z).max())
cb = fig.colorbar(p, ax=ax)

# <headingcell level=2>

# Basic Repeat Map

# <codecell>

raster_width = 11
seq = 'TCTCGTGAACCGTTCTTTCCCGCGGACGTGATGGATGTGGGTGCCTTTATTTGCGACGATATGGTCCGTAAATTAGGTCTCGTTGTTTGTACCCTCTCACTTGGCCGCTTCAACTTTTTTCCGATAATGTCTAATGCACCGACGGAATTATTGTACAGAGTAGCAAGCTCAGGTTGCACGGCAGACCTTGCCGCGTCGGGTCTGCGCACCACCCAAATCTGGGCGCGTCTGGGCCTCGCTGCTACACTGGTTAACCATGCTTCAGACTCTGTGACGATGAAATATCCAAACGACGTTGAATAAAAACGACGGGGAGCGGCGGTGATTTTTATCAATCGCGGTGAAGCAGTTATGCTCGACATCTATTAACAACAGGAGAAAGGCGCCACCGCTCCGGTGTATTATACACTGGGCCGTTTGACCGTCTCATCGACGGGCAACATGACCAAACCGCACATGCATTTCTCGGGCCGAATCGCCCGCGCCTACTGGAAAGCCGGCTCTGGCGATTATGCCGATTTTGAAAGTTTTCTTTCATCCAAAGCGTATATTATTCAGTTTCAAATATCGACCTTGCTGAAGAAAATCAAAGTGATCTTTTTATTTAAAGGCAATGATGGCGATGTACTTAATCGTGCGATCGCTTTGCGGCAGGGCCCCCGTTGGAATAGATTTGATATGCAGGAGCTGTATCCGATCTGGCATATTCTGTCCAATTAACAGCGCAATATCAACGCTGCGCTGTCTCTGCTGGTCGGCGAACACGGACTGATTCAGTCTCCTTTGGCAGGTTTCGTACAAGGTACCACGCTGAGCGCCCTGGGCCAACGGGACTTTGCACTGCGTAAGGACGCAGTGGAAGTGGGCTCCCTGAACCCTGAAGCCGGTGAAGACAAACGTACGACCATCATCTTTACCTATGTACTGCAGCAGCAAGGTTACAAATCCGGTAAATGTTGCGGCGAGGATAAATATGACGTTATTCTGAAAGAAGGGATTATCTACTATACCGTAGTTCTGATCATCCGGGGCTTCAAAGATTCAGACAAGGACGAAGATGACGGACTTAAACATGCGCTTGAAGGATTCGAAGGCGAACGTGGCGCTGCTCTGTCGACTGTAGCATCCGCGTCCGCATGGAGGAGTGGTCAACATAACGGCACCACCCCTTCGTCAAAGGTGGCGCAAGAACTCCGCCAGAAACGCTGCAATTCCAATACAAACATCACCTGCCCACACGTAAACCTTGAACTTAACAAGATATATCGGCTCTTCCCGCTCCAAAACTAAAAGATACCGGACGTGATCGCGATCAGAGGCAAATACTTGACTCATAAGCTGTCAACGGTTGATTTACTGGGTTTTTCTCCGCCAACCTGTCTGCGCTTGCATGATTATGAAGCCGTGTCAGATCCGATGAAAGTGGCGAATTTCCATAACCAGATGGGTTTCTTGGTAGGCGATGCCATCTTCGTTCAGGAACTCATCAAACAGACGGTCGCGCTGATCATTAACAAAGTAAAAAACCCTGGTGGCCTGAAACAGCGAGCCTCAGAAAAACCGAACTCTCAGCTAGTTTGAGGTGGGTCTAATCATGAGCCAGCACTGCGCGACCGTGGGTCTCGTATTCTGGGTGAGCGCGTGCGTGACGATATTCTGTATCTTGTTAACATGGGTTTTAAACATTCGTTCTTGGCTGACCGTGTCATCATGATCAAGATTGAAGAAGAGCTGCATTTTCATACCCAGAGCTACGAGGTCACCTCGCTCGGACAGGGGGTCAGTAATTACCTGGTCACAGCCGATGCGAAAGCCCCAAAACGTCGCCAACTGGCATATCATCTTGGTACTGGGTTCTCATCATTCTACGCTGGGGCGGATGATCAGGCGTCGCGCGTGGAAGTCAAACAGATGCAACGGATCCTGATTGCAGCCGCCCTGCCGGGCCTCCGAAAGAAATTGCGCCTGGATGCACACAATGAATTTATTGTCCCAATCATGACCGAGTTCGACCAGACCGGCCCCTTAACCTTAGGCTACGCATCAGAAAAACGCGCGCTCGATAACATCATGGTGAGTCAGGATTCTGTGCTGGGGAATCTCTTTATGAAATTTTTAGGTGTGCTGGTGGTCGGTATCAGCCGGACAGCGATAGCGGACCCAGATAAGTATATGGCTATTCTGCTGGGTGCGGTTTTCGACATGCTGGCGATGAAAATCATTGAAGTCTTAGATGTTACGTCCAACCGCAACTATTTGACCAATCGCCGTACGACGGAAATCGCAGCTGTGGCAGAAACCTGTGAGGACGGAGCGTTTGTGATGCTGCTGACCACGTGGCTGGGCAAGAAGTCGGATTCCCTGAAGTTCCCTAACTTAGTGATTGTCTATTATATAGTTATGGTCGGCGGCCCGTGCACCGGAGAGCAGCAGAAACGTGCTACAGCAGCCATGAGTAGCGAAATTGCGCTCCAGCCGTATTTCCGCTTCCGCCGGATTGAGCACACTGTCCGCGGCCGCGTCTTTTGACTGGAAAAAAGTTTCGGCGAAGACGCCGGCGATAATCTGGTCTCCAACAAAACCAAACGTCGCGGTAAAGGGCCGCAGTTTAAATATGTGGAACTGGCAGAACTGACCTTAATCAAGCTGTCGATTTGAGGCGGTGTAGCTAACATGGGAGGTAATGCACGTCATGGAATGAAAGGCATTCTGGGTCCGCTGCGCGTTGCCTCTTTAGCTTATCAGGCGAAAGGTGTCATCGGTTTATCTATGTTAAAAAACTGGGCTCCGGCCTAACAAAAAAATCTGCTGTCAGTTGCTGTACTGGTCCCGCTGAGCGCGAGCACAGGGAGCGCCCTGGAAATGGTGCGCGGTCTGAAAGAAGGCAACGCAGTCTTGGTGGCGAAGATGGGGATCGCCAAAGGAGCGACAGGTCGCTGGGCGGCTGTGGCAGATGGTAACGTCGCACCTCCGCTTCGCGAGCAATTAAACTTTCAGGCT'

# <codecell>

seq = 'CTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTACTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTAGTTACTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGCCTTGC'

# <codecell>

#seq[1*x_size : 1*x_size + raster_width]
seq[7:15]

# <codecell>

sum([True, False, True,True])

# <codecell>

raster_width = 11
x_size = 75 # frequency range
y_size = int(len(seq) / raster_width) # number of lines: (cut off the end)
raster_width

# <codecell>

def repeat_score(x, y):
    start_str =  seq[y*raster_width     : (y+1)*raster_width]
    target_str = seq[y*raster_width + x : (y+1)*raster_width + x]
    actual_width = min(len(start_str), len(target_str))
    return sum([start_str[i] == target_str[i] for i in range(actual_width)])

# <codecell>

[[repeat_score(x,y) for x in range(1,x_size-1)] for y in range(y_size-10)]

# <codecell>

X, Y = np.mgrid[:x_size, :y_size]
Z = np.array([[repeat_score(x,y) for x in range(1,x_size+1)] for y in range(y_size)]).T

# <codecell>

fig, ax = plt.subplots()
p = ax.pcolor(X, Y, Z,
              cmap=cm.Greys_r,
              vmin=0, vmax=raster_width)
cb = fig.colorbar(p, ax=ax)

# <codecell>

x, y  = 20, 7
print( seq[y*x_size : y*x_size + raster_width])
print( seq[y*x_size + x : y*x_size + raster_width + x])
sum([start_str[i] == target_str[i] for i in range(raster_width)])

# <headingcell level=3>

# Notes

# <markdowncell>

# I most of the trouble that I had make this was because I am unfamiliar with NumPy arrays and matplotlib.  The lines for Z and p = ax.pcolor(X, Y, Z, cmap=cm.Greys_r, vmin=0, vmax=raster_width) are very sensitive.  The good and the bad of having a graphing platform is that I get scale axes for free.  It will often squish the pixels.  I prefer square pixels.  I need to figure out how to generate a highly non-square graph since the Repeat Map is usually 25 wide x 200 high.

# <headingcell level=1>

# Finished Product

# <codecell>

from Sequence_Utils import debugSequence, weighted_sequence

# <codecell>

class RepeatMap():
    def __init__(self, sequence):
        self.seq = sequence
        self.raster_width = 11
        self.x_size = 25 # frequency range
        self.y_size = int(len(self.seq) / self.raster_width) # number of lines: (cut off the end)

    def repeat_score(self, x, y):
        start_str =  self.seq[y*self.raster_width     : (y+1)*self.raster_width]
        target_str = self.seq[y*self.raster_width + x : (y+1)*self.raster_width + x]
        actual_width = min(len(start_str), len(target_str))
        return sum([start_str[i] == target_str[i] for i in range(actual_width)])

    def render(self):
        X, Y = np.mgrid[:self.x_size, :self.y_size]
        Z = np.array([[self.repeat_score(x,y) for x in range(1,self.x_size+1)] for y in range(self.y_size)]).T
        fig, ax = plt.subplots()
        fig.set_size_inches(self.x_size /10, self.y_size /10)
        p = ax.pcolor(X, Y, Z,
                      cmap=cm.Greys_r,
                      vmin=0, vmax=self.raster_width)
        cb = fig.colorbar(p, ax=ax)
        plt.gca().invert_yaxis()

# <codecell>

rp = RepeatMap(debugSequence(25, 200, 5))
rp.render()

# <codecell>


# -*- coding: utf-8 -*-
"""
Created on Thu Jul 14 21:18:54 2016

@author: charlesgulian
"""

import os
#os.chdir('/Users/annepstein/Work/Deconv')
curr_dir = os.getcwd()
import numpy as np
from astropy.io import fits
import matplotlib.pyplot as plt
import matplotlib

#from photutils import aperture_photometry
#from photutils import CircularAperture

def binImage(pixelArray,M=3,N=3):
    '''
    - Bins pixels along image axes into MxN bins (default MxN = 3x3)
    '''
    pixels = pixelArray
    imgDim1,imgDim2 = np.shape(pixels)
    xBinSize,yBinSize = float(imgDim1)/float(M),float(imgDim2)/float(N)

    imgBinDict = {} # Dictionary for storing
    #print xBinSize,yBinSize

    for i in range(M):
        for j in range(N):
            imgBinDict[i+1,j+1] = pixels[int(np.ceil(i*xBinSize)):int(np.floor((i+1)*xBinSize)),\
                                    int(np.ceil(j*yBinSize)):int(np.floor((j+1)*yBinSize))]
            #print ''
            #print 'Bin: ',i,j
            #print 'Shape: ',np.shape(imgBinDict[i,j])
    return imgBinDict

def computeObjectFlux(x0,y0,radius,image):
    '''
    - Compute total flux within circular aperture of the given radius
        from source at image coordinates (x0,y0)
    '''
    position = (x0,y0)
    aperture = CircularAperture(position,r=radius)
    return aperture_photometry(image,aperture)[0][0]

# getPixels() can be replaced by fits.getdata() (I did not know this)
def getPixels(image_file,delete=False):
    hdulist = fits.open(image_file)
    data = hdulist[0].data
    hdulist.close()
    if delete:
        del hdulist[0].data
    return data

def applyMask(image,mask):
    '''
    - Apply a binary mask to an array
    '''
    masked_image = np.multiply(image,mask)
    return masked_image

def maskImage(image_file,mask_file,masked_image_file=None,Return=False):
    '''
    - Takes a .fits image file and .fits binary mask file as input
    - Applies binary mask to .fits image data
    - Rewrites masked image to new .fits file (masked_image_file)
    '''

    image = fits.getdata(image_file)
    mask = fits.getdata(mask_file)
    masked_image = applyMask(image,mask)
    inds = np.where(masked_image == 0.0)
    masked_image[inds] += 1e-12 # Prevent NaNs

    if masked_image_file == None:
        masked_image_file = image_file.replace('.fits','_masked.fits').replace('Good','MaskedImages').replace('Bad','MaskedImages')

    fits.writeto(masked_image_file,masked_image,fits.getheader(image_file),clobber=True)

    if Return:
        return masked_image

def shift_image(image,x_offset,y_offset):

    # Shifts image pixels from (x,y) to (x-x_offset),(y-y_offset)

    dims = np.shape(image) # Image dimensions
    dim1,dim2 = dims[0],dims[1]

    blank = np.zeros(dims) + 1e-8 # Define blank array to receive new image data
    shifted_image = blank

    dy,dx = x_offset,y_offset # These are intentionally reversed
    for i in range(dim1):
        for j in range(dim2):
            if (i+dx < dim1) and (i+dx >= 0) and (j+dy < dim2) and (j+dy >= 0):
                shifted_image[i,j] = image[i+dx,j+dy] # Why does this work?

    return shifted_image

def subtractBias(image_file,new_image_file=None,bias=0.0,Return=False):
    '''
    - Takes a .fits image file as input
    - Subtracts median from image data, writes new data to new image file (new_image_file)
    '''
    if new_image_file == None:
        new_image_file = image_file

    image = fits.getdata(image_file)
    image -= bias

    fits.writeto(new_image_file,image,fits.getheader(image_file),clobber=True)

    if Return:
        return image

def subtractMedian(image_file,new_image_file=None,Return=False):
    '''
    - Takes a .fits image file as input
    - Subtracts median from image data, writes new data to new image file (new_image_file)
    '''
    if new_image_file == None:
        new_image_file = image_file

    image = fits.getdata(image_file)
    image -= np.median(image)

    fits.writeto(new_image_file,image,fits.getheader(image_file),clobber=True)

    if Return:
        return image

def write_pixel_offset(x_offset,y_offset,image_file,new_image_file=None):
    # Add (x,y) pixel offset to .FITS header of an image

    header = fits.getheader(image_file) # Get original .FITS header
    header['x_offset'] = x_offset # Add new keywords and values to header
    header['y_offset'] = y_offset
    # If no new image file specified, default writes new header to original image header
    if new_image_file == None:
        new_image_file = image_file
    # Write header to new image
    fits.writeto(new_image_file,fits.getdata(image_file),header,clobber=True)

'''
# Testing:

test_image_file = 'AstroImages/Good/fpC-6484-x4078-y134_stitched_alignCropped.fits'
test_image = fits.getdata(test_image_file)

catalog = 'Results/fpC-6484-x4078-y134_stitched_alignCropped_fpC-6484-x4078-y134_stitched_alignCropped_compare.cat'
import sex_stats
fig = sex_stats.data(catalog)
x = fig.get_data('X_IMAGE')
y = fig.get_data('Y_IMAGE')

xlow = np.where(x > 651.0)
xhigh = np.where(x < 658.9)
xin = np.intersect1d(xlow,xhigh)
ylow = np.where(y > 820.0)
yhigh = np.where(y < 826.0)
yin = np.intersect1d(ylow,yhigh)

obj = np.intersect1d(xin,yin)

DATA = fig.Data

x,y = 848.39102,727.23274
radius = 10.

flux = computeObjectFlux(x,y,radius,test_image)
print flux

#testMask = 'AstroImages/Masks/fpC-6484-x4078-y134_stitched_alignCropped_mask.fits'
#maskImage(testImage,testMask)
'''


import os
import cPickle
import matplotlib.pyplot as plt
from datasets import DatasetManager

def plot_cost(results, data_name, plot_label):
    plt.figure(plot_label)
    plt.ylabel('Accuracy (m)', fontsize=30)
    plt.xlabel('Epoch', fontsize=30)
    plt.yscale('symlog')
    plt.tick_params(axis='both', which='major', labelsize=20)
    plt.grid(True)
    for i in range(1, 2, 1):

        y, x = zip(*results[i][data_name])
        name = results[i]['Name']
        plt.plot(x, y, label=name, linewidth=5.0)
    plt.legend(fontsize='xx-large')

def get_metrics(test_set_y, predicted_values, model_name):
    for i in xrange(len(predicted_values)):
        print predicted_values[i][1]

if __name__ == '__main__':
    """
    seed = 50
    with open(os.path.join('experimentation',  'cinvestav_testbed_experiment_results_' + str(seed)), 'rb') as f:
        results = cPickle.load(f)

    plot_cost(
        results=results,
        data_name='cost_train',
        plot_label='Cost on train phase')
    plot_cost(
        results=results,
        data_name='cost_valid',
        plot_label='Cost on valid phase')
    plot_cost(
        results=results,
        data_name='cost_test',
        plot_label='Cost on test phase')
    plt.show()

    """
    seed = 50
    dataset, result = DatasetManager.read_dataset2('test_cleaned_dataset.csv', shared=True, seed=seed)
    with open(os.path.join('trained_models',  'Logistic Regressionbrandeis_university.save'), 'rb') as f:
        model = cPickle.load(f)

    predicted_values = model.predict(dataset)
    get_metrics(
        test_set_y=result,
        predicted_values=predicted_values,
        model_name='Logistic Regression'
    )


import yaml
import pandas as pd
import numpy as np
import sys
import os
from math import sqrt
import matplotlib
import matplotlib.pyplot as plot
import networkx as nx

def get_script_directory():
    """
    This function returns the directory of the script in scrip mode
    In interactive mode returns interpreter name.
    """
    path = os.path.realpath(sys.argv[0])
    if os.path.isdir(path):
        return path
    else:
        return os.path.dirname(path)

def similarity_score(Article1,Article2):
    """
    This function calculates Euclidean distance between to objects
    """
    both_viewed = {}

    for item in dataset[Article1]:
        if item in dataset[Article2]:
            both_viewed[item] = 1

        # The Conditions to check if they both have common rating items
        if len(both_viewed) == 0:
            return 0

        # Finding Euclidean distance
        sum_of_euclidean_distance = []

        for item in dataset[Article1]:
            if item in dataset[Article2]:
                sum_of_euclidean_distance.append(pow(dataset[Article1][item] - dataset[Article2][item], 2))
        sum_of_euclidean_distance = sum(sum_of_euclidean_distance)
        #print (sum_of_euclidean_distance)
        return 1/(1+sqrt(sum_of_euclidean_distance))

def pearson_correlation(Article1,Article2):
    """
    This function calculates Pearson correlation between two vectors
    """
    both_rated = {}
    for item in dataset[Article1]:
        if item in dataset[Article2]:
            both_rated[item] = 1
    number_of_ratings = len(both_rated)
	# Checking for number of ratings in common
    if number_of_ratings == 0:
		return 0
	# Add up all the preferences of each user
    person1_preferences_sum = sum([dataset[Article1][item] for item in both_rated])
    person2_preferences_sum = sum([dataset[Article2][item] for item in both_rated])

    # Sum up the squares of preferences of each user
    person1_square_preferences_sum = sum([pow(dataset[Article1][item],2) for item in both_rated])
    person2_square_preferences_sum = sum([pow(dataset[Article2][item],2) for item in both_rated])

    # Sum up the product value of both preferences for each item
    product_sum_of_both_users = sum([dataset[Article1][item] * dataset[Article2][item] for item in both_rated])

    # Calculate the pearson score
    numerator_value = product_sum_of_both_users - (person1_preferences_sum*person2_preferences_sum/number_of_ratings)
    denominator_value = sqrt((person1_square_preferences_sum - pow(person1_preferences_sum,2)/number_of_ratings) * (person2_square_preferences_sum -pow(person2_preferences_sum,2)/number_of_ratings))
    if denominator_value == 0:
        return 0
    else:
        r = numerator_value/denominator_value
        return r

def find_most_similar_objects(Article1,number_of_users):
	# returns the number_of_users (similar persons) for a given specific person.
	scores = [(pearson_correlation(Article1,other_person),other_person) for other_person in dataset if  other_person != Article1 ]
	# Sort the similar persons so that highest scores person will appear at the first
	scores.sort()
	scores.reverse()
	return (scores[0:number_of_users][0][1])

def get_recommendations(objects, no_of_recommendations):
    """
    This function generates recommendations for specified object
    """
    recommended_articles = []
    input_articles = []
    for article in objects:
        # print (article, find_most_similar_objects(article,2)[0][1], find_most_similar_objects(article,2)[1][1])
        input_articles.append(article)
        recommended_articles.append(find_most_similar_objects(article,no_of_recommendations))
    return input_articles,recommended_articles

# Find the path of script
path = get_script_directory()
print ('Script is located at {}'.format(path))
os.chdir(path)

# import config files
print("Reading configuration")
with open("config.yml", 'r') as ymlfile:
    cfg = yaml.load(ymlfile)

user_ratings_files_path = cfg['project_test_conf']['ratings_file_path']
user_ratings_csv_filename = cfg['project_test_conf']['ratings_file_name']
articles_files_path = cfg['project_test_conf']['articles_file_path']
articles_csv_filename = cfg['project_test_conf']['articles_file_name']
ratings_index = cfg['project_test_conf']['ratings_index_column']
output_file_path = cfg['project_test_conf']['output_path']
output_file_name = cfg['project_test_conf']['output_file_name']
ratings_file = os.path.join(user_ratings_files_path, user_ratings_csv_filename)
articles_file = os.path.join(articles_files_path, articles_csv_filename)
Output_Recommendations = os.path.join(output_file_path, output_file_name)

print("Configuration loaded successfully")
print ('Reading ratings from file {}'.format(ratings_file))
user_ratings = pd.read_csv(ratings_file, index_col=ratings_index)
articles_db = pd.read_csv(articles_file, index_col=ratings_index)
objects_list = list(user_ratings.index)
user_ratings_T = user_ratings.transpose()
dataset = user_ratings_T.to_dict()

# Get recommendations
print('Calculations in progress...')
Article, recommended_article = get_recommendations(objects_list, 5)
print('Calculations completed.')

# Create output files
print('Creating output file')
recommended_article_title = []
for content in recommended_article:
    recommended_article_title.append(articles_db.Title[content])
input_article_title = []
for content in Article:
    input_article_title.append(articles_db.Title[content])

df = pd.DataFrame()
df['Article'] = Article
df['Recommendation'] = recommended_article
df['News'] = input_article_title
df['Recommended_News'] = recommended_article_title
df = df.set_index('Article', drop=True, append=False, inplace=False, verify_integrity=False)
df.to_csv(Output_Recommendations)

print('Output file created.')
print('Check output files at {}'.format(Output_Recommendations))


# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2016, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
This file runs a number of experiments testing the effectiveness of feedback
with noisy inputs.
"""

import os
from copy import deepcopy
import numpy
import cPickle
import matplotlib
import matplotlib.pyplot as plt
import scipy.stats
matplotlib.rcParams['pdf.fonttype'] = 42
plt.ion()

from nupic.data.generators.pattern_machine import PatternMachine
from nupic.data.generators.sequence_machine import SequenceMachine
import feedback_experiment
from feedback_experiment import FeedbackExperiment

def convertSequenceMachineSequence(generatedSequences):
  """
  Convert a sequence from the SequenceMachine into a list of sequences, such
  that each sequence is a list of set of SDRs.
  """
  sequenceList = []
  currentSequence = []
  for s in generatedSequences:
    if s is None:
      sequenceList.append(currentSequence)
      currentSequence = []
    else:
      currentSequence.append(s)

  return sequenceList

def generateSequences(n=2048, w=40, sequenceLength=5, sequenceCount=2,
                      sharedRange=None, seed=42):
  """
  Generate high order sequences using SequenceMachine
  """
  # Lots of room for noise sdrs
  patternAlphabetSize = 10*(sequenceLength * sequenceCount)
  patternMachine = PatternMachine(n, w, patternAlphabetSize, seed)
  sequenceMachine = SequenceMachine(patternMachine, seed)
  numbers = sequenceMachine.generateNumbers(sequenceCount, sequenceLength,
                                            sharedRange=sharedRange )
  generatedSequences = sequenceMachine.generateFromNumbers(numbers)

  return sequenceMachine, generatedSequences, numbers

def sparsenRange(sequenceMachine, sequences, startRange, endRange, probaZero):
  """
  """
  patternMachine = sequenceMachine.patternMachine
  newSequences = []
  for (numseq, s) in enumerate(sequences):
    newSequence = []
    for p,sdr in enumerate(s):
        if p < endRange and p >= startRange:
          newsdr = numpy.array(list(sdr))
          keep = numpy.random.rand(len(newsdr)) > probaZero
          newsdr = newsdr[keep==True]
          newSequence.append(set(newsdr))
        else:
          newSequence.append(sdr)
    newSequences.append(newSequence)

  return newSequences

def crossSequences(sequenceMachine, sequences, pos):
  """
  """
  patternMachine = sequenceMachine.patternMachine
  newSequences = []
  for (numseq, s) in enumerate(sequences):
    newSequence = []
    for p,sdr in enumerate(s):
        if p >= pos:
          newSequence.append(sequences[(numseq +1) % len(sequences)][p])
        else:
          newSequence.append(sdr)
    newSequences.append(newSequence)

  return newSequences

def addTemporalNoise(sequenceMachine, sequences, noiseStart, noiseEnd, noiseProba):
  """
  """
  patternMachine = sequenceMachine.patternMachine
  newSequences = []
  for (numseq, s) in enumerate(sequences):
    newSequence = []
    for p,sdr in enumerate(s):
        if p >= noiseStart and p < noiseEnd:
          newsdr = patternMachine.addNoise(sdr, noiseProba)
          newSequence.append(newsdr)
        else:
          newSequence.append(sdr)
    newSequences.append(newSequence)

  return newSequences

def addPerturbation(sequenceMachine, sequences, noiseType, pos, number=1):
  """
  """
  patternMachine = sequenceMachine.patternMachine
  newSequences = []
  for (numseq, s) in enumerate(sequences):
    newSequence = []
    for p,sdr in enumerate(s):
        if p >= pos and p < pos+number:
          if noiseType == "skip":
            pass
          elif noiseType == "replace":
            newsdr = patternMachine.addNoise(sdr, 1.0)
            newSequence.append(newsdr)
          elif noiseType == "repeat":
            newSequence.append(s[p-1])
          else:
            raise("Unrecognized Noise Type!")
        else:
          newSequence.append(sdr)
    newSequences.append(newSequence)

  return newSequences

def runInference(exp, sequences, enableFeedback=True, apicalTiebreak=True,
                    apicalModulationBasalThreshold=True, inertia=True):
  """
  Run inference on this set of sequences and compute error
  """
  if enableFeedback:
    print "Feedback enabled: "
  else:
    print "Feedback disabled: "

  error = 0
  activityTraces = []
  responses = []
  for i,sequence in enumerate(sequences):
    (avgActiveCells, avgPredictedActiveCells, activityTrace, responsesThisSeq) = exp.infer(
      sequence, sequenceNumber=i, enableFeedback=enableFeedback, apicalTiebreak=apicalTiebreak,
      apicalModulationBasalThreshold=apicalModulationBasalThreshold, inertia=inertia)
    error += avgActiveCells
    activityTraces.append(activityTrace)
    responses.append(responsesThisSeq)
    print " "
  error /= len(sequences)
  print "Average error = ",error
  return error, activityTraces, responses

def runExp(noiseProba, numSequences, nbSeeds, noiseType, sequenceLen, sharedRange, noiseRange, whichPlot, plotTitle):

  allowedNoises = ("skip", "replace", "repeat", "crossover", "pollute")
  if noiseType not in allowedNoises:
    raise(RuntimeError("noiseType must be one of the following: ".join(allowedNoises)))

  meanErrsFB = []; meanErrsNoFB = []; meanErrsNoNoise = []
  stdErrsFB = []; stdErrsNoFB = []; stdErrsNoNoise = []
  meanPerfsFB = []; stdPerfsFB = []
  meanPerfsNoFB = []; stdPerfsNoFB = []
  stdsFB = []
  stdsNoFB=[]
  activitiesFB=[]; activitiesNoFB=[]

  diffsFB = []
  diffsNoFB = []
  overlapsFBL2=[]; overlapsNoFBL2=[]
  overlapsFBL2Next=[]; overlapsNoFBL2Next=[]
  overlapsFBL4=[]; overlapsNoFBL4=[]
  overlapsFBL4Next=[]; overlapsNoFBL4Next=[]
  corrsPredCorrectFBL4=[]; corrsPredCorrectNoFBL4=[]
  diffsFBL4Pred=[]; diffsNoFBL4Pred=[]
  diffsFBL4PredNext=[]; diffsNoFBL4PredNext=[]
  diffsFBL2=[]; diffsNoFBL2=[]
  diffsFBL2Next=[]; diffsNoFBL2Next=[]

  diffsNoAT = []; overlapsNoATL2=[]; overlapsNoATL2Next=[]; overlapsNoATL4=[]
  overlapsNoATL4Next=[]
  corrsPredCorrectNoATL4=[]; diffsNoATL4Pred=[]; diffsNoATL4PredNext=[]
  diffsNoATL2=[]; diffsNoATL2Next=[]
  diffsNoAM = []; overlapsNoAML2=[]; overlapsNoAML2Next=[]; overlapsNoAML4=[]
  overlapsNoAML4Next=[]
  corrsPredCorrectNoAML4=[]; diffsNoAML4Pred=[]; diffsNoAML4PredNext=[]
  diffsNoAML2=[]; diffsNoAML2Next=[]
  diffsNoIN = []; overlapsNoINL2=[]; overlapsNoINL2Next=[]; overlapsNoINL4=[]
  overlapsNoINL4Next=[]
  corrsPredCorrectNoINL4=[]; diffsNoINL4Pred=[]; diffsNoINL4PredNext=[]
  diffsNoINL2=[]; diffsNoINL2Next=[]

  errorsFB=[]; errorsNoFB=[]; errorsNoNoise=[]
  perfsFB = []; perfsNoFB = []

  #for probaZero in probaZeros:
  seed = 42
  for seedx in range(nbSeeds):

      seed = seedx + 123
      profile = False,
      L4Overrides = {"cellsPerColumn": 8}

      numpy.random.seed(seed)

      # Create the sequences and arrays
      print "Generating sequences..."
      sequenceMachine, generatedSequences, numbers = generateSequences(
        sequenceLength=sequenceLen, sequenceCount=numSequences,
        sharedRange=sharedRange,
        seed=seed)

      sequences = convertSequenceMachineSequence(generatedSequences)
      noisySequences = deepcopy(sequences)

      # Apply noise to sequences
      noisySequences = addTemporalNoise(sequenceMachine, noisySequences,
                            noiseStart=noiseRange[0], noiseEnd=noiseRange[1],
                            noiseProba=noiseProba)

      # *In addition* to this, add crossover or single-point noise
      if noiseType == "crossover":
        noisySequences = crossSequences(sequenceMachine, noisySequences,
                                        pos=sequenceLen/2)
      elif noiseType in ("repeat", "replace", "skip"):
        noisySequences = addPerturbation(sequenceMachine, noisySequences,
                                        noiseType=noiseType, pos=sequenceLen/2, number=1)

      inferenceErrors = []

      #Setup experiment and train the network on sequences
      print "Learning sequences..."
      exp = FeedbackExperiment(
        numLearningPasses= 2*sequenceLen,    # To handle high order sequences
        seed=seed,
        L4Overrides=L4Overrides,
      )
      exp.learnSequences(sequences)

      print "Number of columns in exp: ", exp.numColumns
      print "Sequences learned!"

      # Run inference without any noise. This becomes our baseline error
      standardError, activityNoNoise, responsesNoNoise = runInference(exp, sequences)
      inferenceErrors.append(standardError)

      runError, activityFB, responsesFB = runInference(
          exp, noisySequences, enableFeedback=True)
      runError, activityNoFB, responsesNoFB = runInference(
          exp, noisySequences, enableFeedback=False)
      runError, activityNoAT, responsesNoAT = runInference(
          exp, noisySequences, enableFeedback=True, apicalTiebreak=False)
      runError, activityNoAT, responsesNoAM = runInference(
          exp, noisySequences, enableFeedback=True, apicalModulationBasalThreshold=False)
      runError, activityNoIN, responsesNoIN = runInference(
          exp, noisySequences, enableFeedback=True, inertia=False)

      # Now that actual processing is done, we compute various statistics and plot graphs.

      seqlen = len(noisySequences[0])
      sdrlen = 2048 * 8  # Should be the total number of cells in L4. Need to make this more parametrized!
      for numseq in range(len(responsesNoNoise)):

          diffsFB.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesFB[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          diffsNoFB.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoFB[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsFBL2.append( [len(responsesNoNoise[numseq]['L2Responses'][x].intersection(responsesFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoFBL2.append( [len(responsesNoNoise[numseq]['L2Responses'][x].intersection(responsesNoFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsFBL2Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L2Responses'][x].intersection(responsesFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoFBL2Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L2Responses'][x].intersection(responsesNoFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsFBL4.append( [len(responsesNoNoise[numseq]['L4Responses'][x].intersection(responsesFB[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoFBL4.append( [len(responsesNoNoise[numseq]['L4Responses'][x].intersection(responsesNoFB[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsFBL4Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L4Responses'][x].intersection(responsesFB[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoFBL4Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L4Responses'][x].intersection(responsesNoFB[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          diffsFBL4Pred.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesFB[numseq]['L4Predicted'][x])) for x in range(seqlen)] )
          diffsNoFBL4Pred.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoFB[numseq]['L4Predicted'][x])) for x in range(seqlen)] )

          diffsNoAT.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoAT[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoATL2.append( [len(responsesNoNoise[numseq]['L2Responses'][x].intersection(responsesNoAT[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoATL2Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L2Responses'][x].intersection(responsesNoAT[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoATL4.append( [len(responsesNoNoise[numseq]['L4Responses'][x].intersection(responsesNoAT[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoATL4Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L4Responses'][x].intersection(responsesNoAT[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          diffsNoATL4Pred.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoAT[numseq]['L4Predicted'][x])) for x in range(seqlen)] )

          diffsNoAM.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoAM[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoAML2.append( [len(responsesNoNoise[numseq]['L2Responses'][x].intersection(responsesNoAM[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoAML2Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L2Responses'][x].intersection(responsesNoAM[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoAML4.append( [len(responsesNoNoise[numseq]['L4Responses'][x].intersection(responsesNoAM[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoAML4Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L4Responses'][x].intersection(responsesNoAM[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          diffsNoAML4Pred.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoAM[numseq]['L4Predicted'][x])) for x in range(seqlen)] )

          diffsNoIN.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoIN[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoINL2.append( [len(responsesNoNoise[numseq]['L2Responses'][x].intersection(responsesNoIN[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoINL2Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L2Responses'][x].intersection(responsesNoIN[numseq]['L2Responses'][x])) for x in range(seqlen)] )
          overlapsNoINL4.append( [len(responsesNoNoise[numseq]['L4Responses'][x].intersection(responsesNoIN[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          overlapsNoINL4Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L4Responses'][x].intersection(responsesNoIN[numseq]['L4Responses'][x])) for x in range(seqlen)] )
          diffsNoINL4Pred.append( [len(responsesNoNoise[numseq]['L4Responses'][x].symmetric_difference(responsesNoIN[numseq]['L4Predicted'][x])) for x in range(seqlen)] )

          cpcfb = []; cpcnofb=[]; cpcnoat=[]; cpcnoam=[]; cpcnoin=[];
          for x in range(seqlen):
              z1 = numpy.zeros(sdrlen+1); z1[list(responsesNoNoise[numseq]['L4Responses'][x])] = 1; z1[-1] = 1
              z2 = numpy.zeros(sdrlen+1); z2[list(responsesFB[numseq]['L4Predicted'][x])] = 1; z2[-1] = 1
              cpcfb.append(numpy.corrcoef(z1, z2)[0,1])
              z1 = numpy.zeros(sdrlen+1); z1[list(responsesNoNoise[numseq]['L4Responses'][x])] = 1; z1[-1] = 1
              z2 = numpy.zeros(sdrlen+1); z2[list(responsesNoFB[numseq]['L4Predicted'][x])] = 1; z2[-1] = 1
              cpcnofb.append(numpy.corrcoef(z1, z2)[0,1])
              z1 = numpy.zeros(sdrlen+1); z1[list(responsesNoNoise[numseq]['L4Responses'][x])] = 1; z1[-1] = 1
              z2 = numpy.zeros(sdrlen+1); z2[list(responsesNoAT[numseq]['L4Predicted'][x])] = 1; z2[-1] = 1
              cpcnoat.append(numpy.corrcoef(z1, z2)[0,1])
              z1 = numpy.zeros(sdrlen+1); z1[list(responsesNoNoise[numseq]['L4Responses'][x])] = 1; z1[-1] = 1
              z2 = numpy.zeros(sdrlen+1); z2[list(responsesNoAM[numseq]['L4Predicted'][x])] = 1; z2[-1] = 1
              cpcnoam.append(numpy.corrcoef(z1, z2)[0,1])
              z1 = numpy.zeros(sdrlen+1); z1[list(responsesNoNoise[numseq]['L4Responses'][x])] = 1; z1[-1] = 1
              z2 = numpy.zeros(sdrlen+1); z2[list(responsesNoIN[numseq]['L4Predicted'][x])] = 1; z2[-1] = 1
              cpcnoin.append(numpy.corrcoef(z1, z2)[0,1])

          # Note that the correlations are appended across all seeds and sequences
          corrsPredCorrectNoFBL4.append(cpcnofb[1:])
          corrsPredCorrectNoATL4.append(cpcnoat[1:])
          corrsPredCorrectNoINL4.append(cpcnoin[1:])
          corrsPredCorrectNoAML4.append(cpcnoam[1:])
          corrsPredCorrectFBL4.append(cpcfb[1:])

        #   diffsFBL2.append( [len(responsesNoNoise[numseq]['L2Responses'][x].symmetric_difference(responsesFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )
        #   diffsNoFBL2.append( [len(responsesNoNoise[numseq]['L2Responses'][x].symmetric_difference(responsesNoFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )
        #   diffsFBL2Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L2Responses'][x].symmetric_difference(responsesFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )
        #   diffsNoFBL2Next.append( [len(responsesNoNoise[(numseq + 1) % numSequences]['L2Responses'][x].symmetric_difference(responsesNoFB[numseq]['L2Responses'][x])) for x in range(seqlen)] )

          print "Size of L2 responses (FB):", [len(responsesFB[numseq]['L2Responses'][x]) for x in range(seqlen)]
          print "Size of L2 responses (NoNoise):", [len(responsesNoNoise[numseq]['L2Responses'][x]) for x in range(seqlen)]
          print "Size of L4 responses (FB):", [len(responsesFB[numseq]['L4Responses'][x]) for x in range(seqlen)]
          print "Size of L4 responses (NoFB):", [len(responsesNoFB[numseq]['L4Responses'][x]) for x in range(seqlen)]
          print "Size of L4 responses (NoAT):", [len(responsesNoAT[numseq]['L4Responses'][x]) for x in range(seqlen)]
          print "Size of L4 responses (NoAM):", [len(responsesNoAM[numseq]['L4Responses'][x]) for x in range(seqlen)]
          print "Size of L4 responses (NoIN):", [len(responsesNoIN[numseq]['L4Responses'][x]) for x in range(seqlen)]
          print "Size of L4 responses (NoNoise):", [len(responsesNoNoise[numseq]['L4Responses'][x]) for x in range(seqlen)]
          print "Size of L4 predictions (FB):", [len(responsesFB[numseq]['L4Predicted'][x]) for x in range(seqlen)]
          print "Size of L4 predictions (NoFB):", [len(responsesNoFB[numseq]['L4Predicted'][x]) for x in range(seqlen)]
          print "Size of L4 predictions (NoAT):", [len(responsesNoAT[numseq]['L4Predicted'][x]) for x in range(seqlen)]
          print "Size of L4 predictions (NoAM):", [len(responsesNoAM[numseq]['L4Predicted'][x]) for x in range(seqlen)]
          print "Size of L4 predictions (NoIN):", [len(responsesNoIN[numseq]['L4Predicted'][x]) for x in range(seqlen)]
          print "Size of L4 predictions (NoNoise):", [len(responsesNoNoise[numseq]['L4Predicted'][x]) for x in range(seqlen)]
          print "L2 overlap with current (FB): ", overlapsFBL2[-1]
          print "L4 overlap with current (FB): ", overlapsFBL4[-1]
          print "L4 overlap with current (NoFB): ", overlapsNoFBL4[-1]
          print "L4 correlation pred/correct (FB): ", corrsPredCorrectFBL4[-1]
          print "L4 correlation pred/correct (NoFB): ", corrsPredCorrectNoFBL4[-1]
          print "L4 correlation pred/correct (NoAT): ", corrsPredCorrectNoATL4[-1]
          print "L4 correlation pred/correct (NoAM): ", corrsPredCorrectNoATL4[-1]
          print "L4 correlation pred/correct (NoIN): ", corrsPredCorrectNoATL4[-1]
          print "NoNoise sequence:", [list(x)[:2] for x in sequences[numseq]]
          print "Noise sequence:", [list(x)[:2] for x in noisySequences[numseq]]
          print "NoNoise L4 responses:", [list(x)[:2] for x in responsesNoNoise[numseq]['L4Responses']]
          print "NoFB L4 responses:", [list(x)[:2] for x in responsesNoFB[numseq]['L4Responses']]
          print ""

  plt.figure()
  allDataSets = (corrsPredCorrectFBL4, corrsPredCorrectNoFBL4, corrsPredCorrectNoATL4,
        corrsPredCorrectNoAML4, corrsPredCorrectNoINL4)
  allmeans = [numpy.mean(x) for x in allDataSets]
  allstds = [numpy.std(x) for x in allDataSets]
  nbbars = len(allmeans)
  plt.bar(2*(1+numpy.arange(nbbars))-.5, allmeans, 1.0, color='r', edgecolor='none', yerr=allstds, capsize=5, ecolor='k')
  for nn in range(1, nbbars):
      plt.vlines([2, 2 +2*nn], 1.2, 1.2+(nn/10.0), lw=2); plt.hlines(1.2+(nn/10.0), 2, 2+2*nn, lw=2)
      pval = scipy.stats.ranksums(numpy.array(corrsPredCorrectFBL4).ravel(), numpy.array(allDataSets[nn]).ravel())[1]
      if pval > 0.05:
          pvallabel = ' o' #r'$o$'
      elif pval > 0.01:
          pvallabel = '*'
      elif pval > 0.001:
          pvallabel = '**'
      else:
          pvallabel = '***'
      plt.text(3, 1.2+(nn/10.0)+.02, pvallabel, fontdict={"size":14})
  plt.xticks(2*(1+numpy.arange(nbbars)), ('Full', 'No\nFB', 'No Earlier\nFiring', 'No Thresold\nModulation', 'No Slower\nDynamics'))
  plt.ylabel("Avg. Prediction Performance");
  plt.title(plotTitle)
  plt.savefig(plotTitle+".png")
  # scipy.stats.ranksums(numpy.array(corrsPredCorrectFBL4).ravel(), numpy.array(corrsPredCorrectNoATL4).ravel())
  plt.show()
  return (corrsPredCorrectNoFBL4, corrsPredCorrectFBL4, corrsPredCorrectNoATL4, corrsPredCorrectNoAML4, corrsPredCorrectNoINL4)

if __name__ == "__main__":

  plt.ion()

  (corrsPredCorrectNoFBL4, corrsPredCorrectFBL4, corrsPredCorrectNoATL4,
        corrsPredCorrectNoAML4, corrsPredCorrectNoINL4) = runExp(noiseProba=.3,
        numSequences=5, nbSeeds=10, noiseType="pollute", sequenceLen=30, sharedRange=(5,24), noiseRange=(0,30), whichPlot="corrspredcorrect", plotTitle="Individual effects: Continuous noise, shared range")

  (corrsPredCorrectNoFBL4, corrsPredCorrectFBL4, corrsPredCorrectNoATL4,
        corrsPredCorrectNoAML4, corrsPredCorrectNoINL4) = runExp(noiseProba=.3,
        numSequences=5, nbSeeds=10, noiseType="pollute", sequenceLen=30, sharedRange=(0,0), noiseRange=(0,30), whichPlot="corrspredcorrect", plotTitle="Individual effects: Continuous noise, no shared range")

  (corrsPredCorrectNoFBL4, corrsPredCorrectFBL4, corrsPredCorrectNoATL4,
        corrsPredCorrectNoAML4, corrsPredCorrectNoINL4) = runExp(noiseProba=.02,
        numSequences=5, nbSeeds=10, noiseType="replace", sequenceLen=30, sharedRange=(5,24), noiseRange=(0,30), whichPlot="corrspredcorrect", plotTitle="Individual effects: Insert random stimulus, shared range")

  (corrsPredCorrectNoFBL4, corrsPredCorrectFBL4, corrsPredCorrectNoATL4,
        corrsPredCorrectNoAML4, corrsPredCorrectNoINL4) = runExp(noiseProba=.02,
        numSequences=5, nbSeeds=10, noiseType="replace", sequenceLen=30, sharedRange=(0,0), noiseRange=(0,30), whichPlot="corrspredcorrect", plotTitle="Individual effects: Insert random stimulus, no shared range")

  # (corrsPredCorrectNoFBL4, corrsPredCorrectFBL4, corrsPredCorrectNoATL4,
  #      corrsPredCorrectNoAML4, corrsPredCorrectNoINL4) = runExp(noiseProba=.25,
  #      numSequences=5, nbSeeds=10, noiseType="replace", sequenceLen=30, sharedRange=(5,24), noiseRange=(0,30), whichPlot="corrspredcorrect", plotTitle="Individual effects: Random insert + continuous noise, shared range")
  #
  # (corrsPredCorrectNoFBL4, corrsPredCorrectFBL4, corrsPredCorrectNoATL4,
  #      corrsPredCorrectNoAML4, corrsPredCorrectNoINL4) = runExp(noiseProba=.25,
  #      numSequences=5, nbSeeds=10, noiseType="replace", sequenceLen=30, sharedRange=(0,0), noiseRange=(0,30), whichPlot="corrspredcorrect", plotTitle="Individual effects: Random insert + continuous noise, no shared range")


#!/usr/bin/env python
#
# Copyright 2013 Free Software Foundation, Inc.
#
# This file is part of GNU Radio
#
# GNU Radio is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GNU Radio is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNU Radio; see the file COPYING.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street,
# Boston, MA 02110-1301, USA.
#

from gnuradio import gr, filter
from gnuradio import analog
from gnuradio import blocks
from gnuradio import eng_notation
from gnuradio.eng_option import eng_option
from optparse import OptionParser
import sys

try:
    import scipy
except ImportError:
    print "Error: could not import scipy (http://www.scipy.org/)"
    sys.exit(1)

try:
    import pylab
except ImportError:
    print "Error: could not import pylab (http://matplotlib.sourceforge.net/)"
    sys.exit(1)

class example_fir_filter_ccc(gr.top_block):
    def __init__(self, N, fs, bw, tw, atten, D):
        gr.top_block.__init__(self)

        self._nsamps = N
        self._fs = fs
        self._bw = bw
        self._tw = tw
        self._at = atten
        self._decim = D
        taps = filter.firdes.low_pass_2(1, self._fs, self._bw, self._tw, self._at)
        print "Num. Taps: ", len(taps)

        self.src  = analog.noise_source_c(analog.GR_GAUSSIAN, 1)
        self.head = blocks.head(gr.sizeof_gr_complex, self._nsamps)

        self.filt0 = filter.fir_filter_ccc(self._decim, taps)

        self.vsnk_src = blocks.vector_sink_c()
        self.vsnk_out = blocks.vector_sink_c()

        self.connect(self.src, self.head, self.vsnk_src)
        self.connect(self.head, self.filt0, self.vsnk_out)

def main():
    parser = OptionParser(option_class=eng_option, conflict_handler="resolve")
    parser.add_option("-N", "--nsamples", type="int", default=10000,
                      help="Number of samples to process [default=%default]")
    parser.add_option("-s", "--samplerate", type="eng_float", default=8000,
                      help="System sample rate [default=%default]")
    parser.add_option("-B", "--bandwidth", type="eng_float", default=1000,
                      help="Filter bandwidth [default=%default]")
    parser.add_option("-T", "--transition", type="eng_float", default=100,
                      help="Transition band [default=%default]")
    parser.add_option("-A", "--attenuation", type="eng_float", default=80,
                      help="Stopband attenuation [default=%default]")
    parser.add_option("-D", "--decimation", type="int", default=1,
                      help="Decmation factor [default=%default]")
    (options, args) = parser.parse_args ()

    put = example_fir_filter_ccc(options.nsamples,
                                 options.samplerate,
                                 options.bandwidth,
                                 options.transition,
                                 options.attenuation,
                                 options.decimation)
    put.run()

    data_src = scipy.array(put.vsnk_src.data())
    data_snk = scipy.array(put.vsnk_out.data())

    # Plot the signals PSDs
    nfft = 1024
    f1 = pylab.figure(1, figsize=(12,10))
    s1 = f1.add_subplot(1,1,1)
    s1.psd(data_src, NFFT=nfft, noverlap=nfft/4,
           Fs=options.samplerate)
    s1.psd(data_snk, NFFT=nfft, noverlap=nfft/4,
           Fs=options.samplerate)

    f2 = pylab.figure(2, figsize=(12,10))
    s2 = f2.add_subplot(1,1,1)
    s2.plot(data_src)
    s2.plot(data_snk.real, 'g')

    pylab.show()

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        pass


"""
This is a set of classes to perform fast (approximate) nearest neighbors
searches over Hamming spaces.

[1] M. Charikar. Similarity Estimation Techniques from Rounding Algorithms.
    ACM Symposium on Theory of Computing, 2002.
"""
__all__ = ["HammingANN", "HammingBrute", "HammingBallTree"]

import numpy as np
from scipy.spatial import distance
from sklearn.neighbors import BallTree
from .utils import create_rng, packbits_axis, unpackbits_axis, hamming_cdist

class HammingSearchBase(object):
    """Base class for Hamming neighbors search"""
    def fit(self, X):
        raise NotImplementedError('HammingSearchBase.fit')

    def query(self, X, k, return_dist=False):
        raise NotImplementedError('HammingSearchBase.query')

    @staticmethod
    def _validate_input(X, return_compact=True):
        X = np.atleast_2d(np.asarray(X, dtype=np.uint8))
        if X.ndim != 2:
            raise ValueError("Input hamming array must be two dimensions")
        if return_compact:
            return packbits_axis(X)
        else:
            X[X != 0] = 1
            return X

class HammingBrute(HammingSearchBase):
    def __init__(self, compact=False):
        self.compact = compact

    def fit(self, X):
        """Fit a set of hamming vectors

        Parameters
        ----------
        X : array_like
            an array of size (n_features, n_bits). Nonzero entries will be
            evaluated as 1, and zero entries as 0
        """
        if self.compact:
            self._fit_X = self._validate_input(X)
        else:
            self._fit_X = self._validate_input(X, False)
        return self

    def query(self, X, k, return_dist=False):
        if self.compact:
            X = self._validate_input(X)
            cdist = hamming_cdist(X, self._fit_X)
        else:
            X = self._validate_input(X, False)
            cdist = distance.cdist(X, self._fit_X, 'hamming')
        ind = np.argsort(cdist, 1)[:, :k]
        if return_dist:
            rows = np.arange(ind.shape[0])[:, np.newaxis]
            dist = cdist[rows, ind]
            if not self.compact:
                dist = (dist * X.shape[1]).astype(int)
            return ind, dist
        else:
            return ind

class HammingBallTree(HammingSearchBase):
    def __init__(self, leaf_size=40, query_kwds=None):
        self.leaf_size = leaf_size
        self.query_kwds = query_kwds or {}

    def fit(self, X):
        X = self._validate_input(X, return_compact=False)
        self._tree = BallTree(X, metric='hamming', leaf_size=self.leaf_size)
        return self

    def query(self, X, k, return_dist=False):
        X = self._validate_input(X, return_compact=False)

        if return_dist:
            dist, ind = self._tree.query(X, k, return_distance=True)
            return ind, (dist * X.shape[1]).astype(int)
        else:
            return self._tree.query(X, k, return_distance=False)

class HammingANN(HammingSearchBase):
    def __init__(self, epsilon=0.5, random_state=None):
        self.epsilon = epsilon
        self.random_state = random_state

    def fit(self, X):
        """Fit a set of hamming vectors

        Parameters
        ----------
        X : array_like
            an array of size (n_features, n_bits). Nonzero entries will be
            evaluated as 1, and zero entries as 0
        """
        self._X_fit = self._validate_input(X, False)
        self._X_fit_compact = packbits_axis(self._X_fit)
        N, n_bits = self._X_fit.shape

        # choose number of permutations based on epsilon
        M = 2 * int(np.ceil(N ** (1. / (1. + self.epsilon))))

        rng = create_rng(self.random_state)
        P_indices = np.array([rng.choice(n_bits, n_bits, replace=False)
                              for i in range(M)])

        # P_compact will be of shape (M, X.shape[0]), and contains
        # M bit-permutations applied across all the keys
        P = self._X_fit[:, P_indices]
        P_compact = packbits_axis(P).T

        # Do a lexicographic sort of all the permuted bits.
        # Here's where cython would help immensely. We could store just
        # M permutation-bit arrays, and write a custom sort & binary search
        # which will work on these permutations and orderings.
        sort_indices = np.argsort(P_compact, 1)
        P_compact_sorted = P_compact[np.arange(M)[:, None], sort_indices]
        unsort_indices = np.argsort(sort_indices, 1)

        #----------------- just a sanity check (TODO: REMOVE THIS)
        reordered = P_compact_sorted[np.arange(M)[:, np.newaxis],
                                     unsort_indices]
        assert np.all(reordered == P_compact)
        #---------------------------------------------------------

        self._sort_indices = sort_indices
        self._unsort_indices = unsort_indices
        self._P_compact_sorted = P_compact_sorted
        return self

    def query(self, X, k, return_dist=False):
        """Query a set of distances

        Parameters
        ----------
        X : array_like
           an [n_samples, n_bits] array of hamming features. These will be
           interpreted as zeros and ones.
        """
        X_compact = self._validate_input(X)

        nbrs = np.zeros([X_compact.shape[0], k], dtype=int)

        if return_dist:
            dist = np.zeros_like(nbrs)

        M, N = self._P_compact_sorted.shape

        # TODO: MAKE THIS MORE EFFICIENT
        for i, val in enumerate(X_compact):
            # find ordered index within each random permutation
            P_indices = np.array([np.searchsorted(self._P_compact_sorted[j],
                                                  val) for j in range(M)])

            # get upper/lower indices within each permutation
            ind_uplo = np.clip(np.vstack([P_indices, P_indices + 1]), 0, N-1)

            # from indices within the sorted permutations, find the
            # unique set of indices from the original set of hashes
            ind_to_check = np.unique(self._sort_indices[range(M), ind_uplo])

            # compute hamming distances for these points, and put into results
            distances = hamming_cdist(val, self._X_fit_compact[ind_to_check])
            nearest = np.argsort(distances[0])[:k]
            nbrs[i, :len(nearest)] = ind_to_check[nearest]
            if return_dist:
                dist[i, :len(nearest)] = distances[0, nearest[:k]]

        if return_dist:
            return nbrs, dist
        else:
            return nbrs


# -*- coding: utf-8 -*-
#
# test_3d.py
#
# This file is part of NEST.
#
# Copyright (C) 2004 The NEST Initiative
#
# NEST is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# NEST is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with NEST.  If not, see <http://www.gnu.org/licenses/>.

"""
A spatial network in 3D
-------------------------

Hans Ekkehard Plesser, UMB
"""

import nest
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

nest.ResetKernel()

pos = nest.spatial.free(nest.random.uniform(-0.5, 0.5), extent=[1.5, 1.5, 1.5])

l1 = nest.Create('iaf_psc_alpha', 1000, positions=pos)

# visualize

# extract position information, transpose to list of x, y and z positions
xpos, ypos, zpos = zip(*nest.GetPosition(l1))
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(xpos, ypos, zpos, s=15, facecolor='b')

# full connections in box volume [-0.2,0.2]**3
nest.Connect(l1, l1,
             {'rule': 'pairwise_bernoulli',
              'p': 1.,
              'allow_autapses': False,
              'mask': {'box': {'lower_left': [-0.2, -0.2, -0.2],
                               'upper_right': [0.2, 0.2, 0.2]}}})

# show connections from center element
# sender shown in red, targets in green
ctr = nest.FindCenterElement(l1)
xtgt, ytgt, ztgt = zip(*nest.GetTargetPositions(ctr, l1)[0])
xctr, yctr, zctr = nest.GetPosition(ctr)
ax.scatter([xctr], [yctr], [zctr], s=40, facecolor='r')
ax.scatter(xtgt, ytgt, ztgt, s=40, facecolor='g', edgecolor='g')

tgts = nest.GetTargetNodes(ctr, l1)[0]
distances = nest.Distance(ctr, l1)
tgt_distances = [d for i, d in enumerate(distances) if i + 1 in tgts]

plt.figure()
plt.hist(tgt_distances, 25)
plt.show()