import numpy as np

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import ignore_warnings

from sklearn.decomposition import DictionaryLearning
from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.decomposition import SparseCoder
from sklearn.decomposition import dict_learning_online
from sklearn.decomposition import sparse_encode

rng_global = np.random.RandomState(0)
n_samples, n_features = 10, 8
X = rng_global.randn(n_samples, n_features)

def test_dict_learning_shapes():
    n_components = 5
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_overcomplete():
    n_components = 12
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_reconstruction():
    n_components = 12
    dico = DictionaryLearning(n_components, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

    # used to test lars here too, but there's no guarantee the number of
    # nonzero atoms is right.

def test_dict_learning_nonzero_coefs():
    n_components = 4
    dico = DictionaryLearning(n_components, transform_algorithm='lars',
                              transform_n_nonzero_coefs=3, random_state=0)
    code = dico.fit(X).transform(X[1])
    assert_true(len(np.flatnonzero(code)) == 3)

    dico.set_params(transform_algorithm='omp')
    code = dico.transform(X[1])
    assert_equal(len(np.flatnonzero(code)), 3)

def test_dict_learning_unknown_fit_algorithm():
    n_components = 5
    dico = DictionaryLearning(n_components, fit_algorithm='<unknown>')
    assert_raises(ValueError, dico.fit, X)

def test_dict_learning_split():
    n_components = 5
    dico = DictionaryLearning(n_components, transform_algorithm='threshold',
                              random_state=0)
    code = dico.fit(X).transform(X)
    dico.split_sign = True
    split_code = dico.transform(X)

    assert_array_equal(split_code[:, :n_components] -
                       split_code[:, n_components:], code)

def test_dict_learning_online_shapes():
    rng = np.random.RandomState(0)
    n_components = 8
    code, dictionary = dict_learning_online(X, n_components=n_components,
                                            alpha=1, random_state=rng)
    assert_equal(code.shape, (n_samples, n_components))
    assert_equal(dictionary.shape, (n_components, n_features))
    assert_equal(np.dot(code, dictionary).shape, X.shape)

def test_dict_learning_online_verbosity():
    n_components = 5
    # test verbosity
    from sklearn.externals.six.moves import cStringIO as StringIO
    import sys

    old_stdout = sys.stdout
    try:
        sys.stdout = StringIO()
        dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=1,
                                           random_state=0)
        dico.fit(X)
        dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=2,
                                           random_state=0)
        dico.fit(X)
        dict_learning_online(X, n_components=n_components, alpha=1, verbose=1,
                             random_state=0)
        dict_learning_online(X, n_components=n_components, alpha=1, verbose=2,
                             random_state=0)
    finally:
        sys.stdout = old_stdout

    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_estimator_shapes():
    n_components = 5
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, random_state=0)
    dico.fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_overcomplete():
    n_components = 12
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20,
                                       random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_initialization():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)
    dico = MiniBatchDictionaryLearning(n_components, n_iter=0,
                                       dict_init=V, random_state=0).fit(X)
    assert_array_equal(dico.components_, V)

def test_dict_learning_online_partial_fit():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    dict1 = MiniBatchDictionaryLearning(n_components, n_iter=10*len(X),
                                        batch_size=1,
                                        alpha=1, shuffle=False, dict_init=V,
                                        random_state=0).fit(X)
    dict2 = MiniBatchDictionaryLearning(n_components, alpha=1,
                                        n_iter=1, dict_init=V,
                                        random_state=0)
    for i in range(10):
        for sample in X:
            dict2.partial_fit(sample)

    assert_true(not np.all(sparse_encode(X, dict1.components_, alpha=1) ==
                           0))
    assert_array_almost_equal(dict1.components_, dict2.components_,
                              decimal=2)

def test_sparse_encode_shapes():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'):
        code = sparse_encode(X, V, algorithm=algo)
        assert_equal(code.shape, (n_samples, n_components))

def test_sparse_encode_error():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = sparse_encode(X, V, alpha=0.001)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)

def test_sparse_encode_error_default_sparsity():
    rng = np.random.RandomState(0)
    X = rng.randn(100, 64)
    D = rng.randn(2, 64)
    code = ignore_warnings(sparse_encode)(X, D, algorithm='omp',
                                          n_nonzero_coefs=None)
    assert_equal(code.shape, (100, 2))

def test_unknown_method():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    assert_raises(ValueError, sparse_encode, X, V, algorithm="<unknown>")

def test_sparse_coder_estimator():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = SparseCoder(dictionary=V, transform_algorithm='lasso_lars',
                       transform_alpha=0.001).transform(X)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)


"""
Testing for the nearest centroid module.
"""

import numpy as np
from scipy import sparse as sp
from numpy.testing import assert_array_equal
from numpy.testing import assert_equal

from sklearn.neighbors import NearestCentroid
from sklearn import datasets
from sklearn.metrics.pairwise import pairwise_distances

# toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
X_csr = sp.csr_matrix(X)  # Sparse matrix
y = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
T_csr = sp.csr_matrix(T)
true_result = [-1, 1, 1]

# also load the iris dataset
# and randomly permute it
iris = datasets.load_iris()
rng = np.random.RandomState(1)
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

def test_classification_toy():
    # Check classification on a toy dataset, including sparse versions.
    clf = NearestCentroid()
    clf.fit(X, y)
    assert_array_equal(clf.predict(T), true_result)

    # Same test, but with a sparse matrix to fit and test.
    clf = NearestCentroid()
    clf.fit(X_csr, y)
    assert_array_equal(clf.predict(T_csr), true_result)

    # Fit with sparse, test with non-sparse
    clf = NearestCentroid()
    clf.fit(X_csr, y)
    assert_array_equal(clf.predict(T), true_result)

    # Fit with non-sparse, test with sparse
    clf = NearestCentroid()
    clf.fit(X, y)
    assert_array_equal(clf.predict(T_csr), true_result)

    # Fit and predict with non-CSR sparse matrices
    clf = NearestCentroid()
    clf.fit(X_csr.tocoo(), y)
    assert_array_equal(clf.predict(T_csr.tolil()), true_result)

def test_precomputed():
    clf = NearestCentroid(metric="precomputed")
    clf.fit(X, y)
    S = pairwise_distances(T, clf.centroids_)
    assert_array_equal(clf.predict(S), true_result)

def test_iris():
    # Check consistency on dataset iris.
    for metric in ('euclidean', 'cosine'):
        clf = NearestCentroid(metric=metric).fit(iris.data, iris.target)
        score = np.mean(clf.predict(iris.data) == iris.target)
        assert score > 0.9, "Failed with score = " + str(score)

def test_iris_shrinkage():
    # Check consistency on dataset iris, when using shrinkage.
    for metric in ('euclidean', 'cosine'):
        for shrink_threshold in [None, 0.1, 0.5]:
            clf = NearestCentroid(metric=metric,
                                  shrink_threshold=shrink_threshold)
            clf = clf.fit(iris.data, iris.target)
            score = np.mean(clf.predict(iris.data) == iris.target)
            assert score > 0.8, "Failed with score = " + str(score)

def test_pickle():
    import pickle

    # classification
    obj = NearestCentroid()
    obj.fit(iris.data, iris.target)
    score = obj.score(iris.data, iris.target)
    s = pickle.dumps(obj)

    obj2 = pickle.loads(s)
    assert_equal(type(obj2), obj.__class__)
    score2 = obj2.score(iris.data, iris.target)
    assert_array_equal(score, score2,
                       "Failed to generate same score"
                       " after pickling (classification).")

def test_shrinkage_threshold_decoded_y():
    clf = NearestCentroid(shrink_threshold=0.01)
    y_ind = np.asarray(y)
    y_ind[y_ind == -1] = 0
    clf.fit(X, y_ind)
    centroid_encoded = clf.centroids_
    clf.fit(X, y)
    assert_array_equal(centroid_encoded, clf.centroids_)

def test_predict_translated_data():
    # Test that NearestCentroid gives same results on translated data

    rng = np.random.RandomState(0)
    X = rng.rand(50, 50)
    y = rng.randint(0, 3, 50)
    noise = rng.rand(50)
    clf = NearestCentroid(shrink_threshold=0.1)
    clf.fit(X, y)
    y_init = clf.predict(X)
    clf = NearestCentroid(shrink_threshold=0.1)
    X_noise = X + noise
    clf.fit(X_noise, y)
    y_translate = clf.predict(X_noise)
    assert_array_equal(y_init, y_translate)

def test_manhattan_metric():
    # Test the manhattan metric.

    clf = NearestCentroid(metric='manhattan')
    clf.fit(X, y)
    dense_centroid = clf.centroids_
    clf.fit(X_csr, y)
    assert_array_equal(clf.centroids_, dense_centroid)
    assert_array_equal(dense_centroid, [[-1, -1], [1, 1]])


import numpy as np
import operator
from sklearn import naive_bayes
from sklearn import svm, tree

from kernels import GaussianKernel

class AbstractModel(object):
    """Abstract model of a learning algorithm. When implementing a subclass,
    you have to implement method ``train``. Method ``predict`` is implemented
    by default.
    """

    def __init__(self, feature_selector):
        """Default initializer requires only feature selector to be specified.
        If you want to add additional parameters to your implementation of the
        model, be sure to call this initializer in your initializer first.

        :param feature_selector: Feature selector class from package
               ``selectors``.
        """

        self.feature_selector = feature_selector

    def train(self, documents):
        """Trains the model on the provided list of **labeled** documents.
        This method is expected to initialize some sort of predictor field(s)
        that will be used by method ``predict``, e.g. in Naive Bayes model,
        the initialized fields could be ``prior`` and ``likelihood``.

        :param documents: Labeled documents used to train the predictor.
        """

        raise NotImplementedError()

    def predict(self, document, n=1):
        """Predicts label(s) for given document.
        Note that before running this method, method ``train`` must be run.

        :param document: Document to be labeled.
        :param n: Number of predictions, 1 by default.
        :returns: Predicted label(s) of the document in descending order.
        """

        raise NotImplementedError()

class BaselineModel(AbstractModel):
    """This baseline model always predict label that is the most frequent."""

    def __init__(self, feature_selector):
        super(BaselineModel, self).__init__(feature_selector)

    def train(self, documents):
        X, Y = self.feature_selector.build(documents)
        labels_freq = {}
        for document in documents:
            if document.label in labels_freq:
                labels_freq[document.label] += 1
            else:
                labels_freq[document.label] = 1

        self.labels_freq = sorted(labels_freq.items(), reverse=True,
                                  key=operator.itemgetter(1))

    def predict(self, document, n=1):
        top_n = self.labels_freq[:n]
        labels = map(lambda x: x[0], top_n)
        return labels

class NaiveBayesModel(AbstractModel):
    """Naive Bayes model. No +1 smoothing is used in this model, the selector
    is expected to remove words that are not in the vocabulary.
    """

    def __init__(self, feature_selector):
        super(NaiveBayesModel, self).__init__(feature_selector)

    def train(self, documents):
        X, Y = self.feature_selector.build(documents)
        nb = naive_bayes.GaussianNB()
        nb.fit(X, np.concatenate(Y))
        self.nb = nb

    def predict(self, document, n=1):
        x = self.feature_selector.get_x(document)
        probs = self.nb.predict_proba([x])[0]
        Y = probs.argsort()[::-1]
        labels = map(self.feature_selector.get_label, Y)

        return labels[:n]

    def __str__(self):
        return "NaiveBayesModel(feature_selector=%s)" \
            % self.feature_selector

class SVMModel(AbstractModel):
    """Support Vector Machine model."""

    def __init__(self, feature_selector, kernel=GaussianKernel(), C=1,
                 cache_size=200):
        super(SVMModel, self).__init__(feature_selector)
        self.C = C
        self.kernel = kernel
        self.cache_size = cache_size

    def train(self, documents):
        X, Y = self.feature_selector.build(documents)
        if hasattr(self.kernel, 'sklearn_name'):
            self.svm = svm.SVC(C=self.C, kernel=self.kernel.sklearn_name,
                               probability=True, cache_size=self.cache_size,
                               **self.kernel.sklearn_params)
        else:
            self.svm = svm.SVC(C=self.C, kernel=self.kernel.compute)
        self.svm.fit(X, np.concatenate(Y))

    def predict(self, document, n=1):
        x = self.feature_selector.get_x(document)
        probs = self.svm.predict_proba([x])[0]
        Y = probs.argsort()[::-1]
        labels = map(self.feature_selector.get_label, Y)

        return labels[:n]

    def __str__(self):
        return "SVMModel(feature_selector=%s, kernel=%s, C=%s)" \
            % (self.feature_selector, self.kernel, self.C)

class CARTModel(AbstractModel):
    "Decision Tree Model using CART algorithm."

    def __init__(self, feature_selector):
        super(CARTModel, self).__init__(feature_selector)

    def train(self, documents):
        X, Y = self.feature_selector.build(documents)
        self.clf = tree.DecisionTreeClassifier()
        self.clf.fit(X, np.concatenate(Y))

    def predict(self, document, n=1):
        x = self.feature_selector.get_x(document)
        probs = self.clf.predict_proba([x])[0]
        Y = probs.argsort()[::-1]
        labels = map(self.feature_selector.get_label, Y)

        return labels[:n]

    def __str__(self):
        return "CARTModel(feature_selector=%s)" % self.feature_selector


from sklearn.datasets import fetch_20newsgroups
import os, sys
import subprocess, random
import unicodedata

base = "/cab1/corpora/bayes_opt/20_newsgroups/"

groups = {'all_topics':['talk.religion.misc', 'comp.windows.x', 'rec.sport.baseball', 'talk.politics.mideast', 'comp.sys.mac.hardware', 'sci.space', 'talk.politics.guns', 'comp.graphics', 'comp.os.ms-windows.misc', 'soc.religion.christian', 'talk.politics.misc', 'rec.motorcycles', 'comp.sys.ibm.pc.hardware', 'rec.sport.hockey', 'misc.forsale', 'sci.crypt', 'rec.autos', 'sci.med', 'sci.electronics', 'alt.atheism'],
          'science':['sci.space', 'sci.crypt', 'sci.med', 'sci.electronics'],
          'religion':['talk.religion.misc', 'alt.atheism'],
          'comp':['comp.windows.x', 'comp.graphics']}
random.seed(999)

def do_stuff(train_or_test):
    make_dev = train_or_test == 'train'
    for group in groups:
        #make directory
        os.system('mkdir -p ' + base + group)

        examples = {}
        newsgroups_data = fetch_20newsgroups(subset=train_or_test, categories=groups[group], remove=('headers'))
        for i in range(len(newsgroups_data['data'])):
            line = newsgroups_data['data'][i]
#            line = unicode(line, errors='ignore')
            unicodedata.normalize('NFKD', line).encode('ascii', 'ignore')
            line = ''.join(ch for ch in line if unicodedata.category(ch)[0]!="C")
            contents = line.replace('\n', ' ').replace('\r', ' ').replace('\t', ' ').replace('\0',' ').replace('*', '.').strip().replace('"', "''").replace("\\","\\\\")

            examples[len(examples)] = (contents, newsgroups_data['target_names'][newsgroups_data['target'][i]])

        dev_items = sorted(random.sample(xrange(len(examples)), int(.2*len(examples))))
        if not make_dev:
            dev_items = []
        print(len(dev_items))

        if make_dev:
            train_json_out = open(base + group + '/train.json', 'w')
            train_csv_out = open(base + group + '/train.csv', 'w')
            dev_json_out = open(base + group + '/dev.json', 'w')
            dev_csv_out = open(base + group + '/dev.csv', 'w')
        else:
            train_json_out = open(base + group + '/test.json', 'w')
            train_csv_out = open(base + group + '/test.csv', 'w')

        train_json_out.write('{\n')
        train_csv_out.write('idontknow,whattoputhere\n')
        if make_dev:
            dev_csv_out.write('idontknow,whattoputhere\n')
            dev_json_out.write('{\n')

        dev_counter = 0
        train_counter = 0
        for index in examples:
            if index in dev_items:
                dev_counter = dev_counter + 1
                if index == dev_items[len(dev_items)-1]:
                    dev_json_out.write('  "' + str(dev_counter) + '": "' + examples[index][0].replace('\*','*').encode('ascii', 'ignore') + '"\n')
                else:
                    dev_json_out.write('  "' + str(dev_counter) + '": "' + examples[index][0].replace('\*','*').encode('ascii', 'ignore') + '",\n')
                dev_csv_out.write(str(dev_counter) + ',' + str(examples[index][1]) + '\n')
            else:
                train_counter = train_counter + 1
                train_json_out.write('  "' + str(train_counter) + '": "' + examples[index][0].replace('\*','*').encode('ascii', 'ignore') + '",\n')
                train_csv_out.write(str(train_counter) + ',' + str(examples[index][1]) + '\n')

        train_json_out.write('}')
        if make_dev:
            dev_json_out.write('}')
        print("DON'T FORGET TO REMOVE THE EXTRA COMMA AT THE END OF TRAIN.JSON AND TEST.JSON")

        train_json_out.close()
        train_csv_out.close()
        if make_dev:
            dev_json_out.close()
            dev_csv_out.close()

do_stuff('train')
do_stuff('test')
#dev_items = sorted(random.sample(xrange(len(examples)), int(.2*len(examples))))


#!/usr/bin/python

import analysis
import os
import sys
import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt

def eval_dict( s ):
   ret = None
   try:
      exec("ret = " + s)
   except:
      return None

   return ret

def cdf_compare( dists, title, xl, xr, yl, yr, labels ):
   mm = min(dists[0])
   ma = max(dists[0])
   cnt = len(dists[0])
   for i in xrange(1,len(dists)):
      mm = min( mm, min(dists[i]) )
      ma = max( ma, max(dists[i]) )
      cnt = min( cnt, len(dists[i]) )

   print "cnt = " + str(cnt)
   x = np.linspace( mm, ma, cnt )

   i = 0
   for dist in dists:
      ecdf = sm.distributions.ECDF( dist )
      plt.step( x, ecdf(x), label=labels[i] )
      i += 1

      dist.sort()
      #print dist

   plt.title( title )
   plt.xticks( xr )
   plt.yticks( yr )
   plt.xlabel( xl )
   plt.ylabel( yl )
   plt.legend( labels, loc=4 )
   plt.show()

if __name__ == "__main__":
   syndicate_data_1k = {}
   syndicate_data_1M = {}
   syndicate_data_50M = {}

   s3_data_20k = {}
   s3_data_50M = {}
   s3_data_100blk = {}
   s3_data_100blk_nocache = {}
   plc_data_100blk = {}
   syndicate_data_100blk = {}

   intersection = []

   for expfile in os.listdir( sys.argv[1] ):
      expfd = open( os.path.join( sys.argv[1], expfile ), "r" )
      expdata = analysis.parse_experiments( expfd )
      expfd.close()

      if len(expdata['fcdistro']) > 0 and "12" not in expdata['fcdistro']:
         print >> sys.stderr, "%s: wrong distro '%s'" % (expfile, expdata['fcdistro'])
         continue

      syndicate_exp_1k = analysis.read_experiment_data( expdata, "Nr1w-x5-small-syndicate.py" )
      syndicate_exp_1M = analysis.read_experiment_data( expdata, "Nr1w-x5-1M-syndicate.py" )
      syndicate_exp_50M = analysis.read_experiment_data( expdata, "Nr1w-x5-50M-syndicate-4.py" )
      syndicate_exp_100blk = analysis.read_experiment_data( expdata, "Nr1w-syndicate-3.py" )

      s3_exp_20k = analysis.read_experiment_data( expdata, "Nr1w-x5.py" )
      s3_exp_100blk = analysis.read_experiment_data( expdata, "Nr1w-x5-100blk-s3-cache-chunked.py" )
      plc_exp_100blk = analysis.read_experiment_data( expdata, "Nr1w-x5-100blk-planetlab-cache-chunked.py" )
      s3_exp_50M = analysis.read_experiment_data( expdata, "Nr1w-x5-50M.py" )

      s3_exp_100blk_nocache = analysis.read_experiment_data( expdata, "Nr1w-x5-100blk-s3-chunked.py" )

      intersect = True

      """
      if syndicate_exp_1k != None and len(syndicate_exp_1k) > 0 and syndicate_exp_1k[0] != None:
         syndicate_data_1k[expfile] = eval_dict( syndicate_exp_1k[0][0] )
      else:
         intersect = False

      if syndicate_exp_1M != None and len(syndicate_exp_1M) > 0 and syndicate_exp_1M[0] != None:
         syndicate_data_1M[expfile] = eval_dict( syndicate_exp_1M[0][0] )
      else:
         intersect = False

      if syndicate_exp_50M != None and len(syndicate_exp_50M) > 0 and syndicate_exp_50M[0] != None:
         syndicate_data_50M[expfile] = eval_dict( syndicate_exp_50M[0][0] )
      else:
         intersect = False

      if s3_exp_20k != None and len(s3_exp_20k) > 0 and s3_exp_20k[0] != None:
         s3_data_20k[expfile] = eval_dict( s3_exp_20k[0][0] )
      else:
         intersect = False

      if s3_exp_50M != None and len(s3_exp_50M) > 0 and s3_exp_50M[0] != None:
         s3_data_50M[expfile] = eval_dict( s3_exp_50M[0][0] )
      else:
         intersect = False
      """

      if s3_exp_100blk != None and len(s3_exp_100blk) > 0 and s3_exp_100blk[0] != None:
         s3_data_100blk[expfile] = eval_dict( s3_exp_100blk[0][0] )
      else:
         intersect = False

      if plc_exp_100blk != None and len(plc_exp_100blk) > 0 and plc_exp_100blk[-1] != None:
         plc_data_100blk[expfile] = eval_dict( plc_exp_100blk[-1][0] )
      else:
         intersect = False

      if s3_exp_100blk_nocache != None and len(s3_exp_100blk_nocache) > 0 and s3_exp_100blk_nocache[-1] != None:
         s3_data_100blk_nocache[expfile] = eval_dict( s3_exp_100blk_nocache[-1][0] )
      else:
         intersect = False

      if syndicate_exp_100blk != None and len(syndicate_exp_100blk) > 0 and syndicate_exp_100blk[-1] != None:
         syndicate_data_100blk[expfile] = eval_dict( syndicate_exp_100blk[-1][0] )
      else:
         intersect = False

      if intersect:
         intersection.append( expfile )

   for expfile in os.listdir( sys.argv[1] ):
      if expfile not in intersection:
         print >> sys.stderr, "Node %s did not pass all tests" % expfile

   print >> sys.stderr, "%s nodes have data" % len(intersection)

   syndicate = { 'first_1k': [], 'last_1k': [], 'first_1m': [], 'last_1m': [], 'first_50m': [], 'last_50m': [], 'first_100blk': [], 'last_100blk': [] }
   s3 = { 'first_20k': [], 'last_20k': [], 'first_50m': [], 'last_50m': [], 'first_100blk': [], 'last_100blk': [], 'first_100blk_nocache': [], 'last_100blk_nocache': [] }
   plc = {'first_100blk' : [], 'last_100blk': [] }

   num_valid = 0
   slow = []
   for node in intersection:
      valid = True
      #data_list = [("syndicate 1k", syndicate_data_1k), ("syndicate 1M", syndicate_data_1M), ("syndicate 50M", syndicate_data_50M), ("S3 20k", s3_data_20k), ("S3 50M", s3_data_50M), ("S3 100blk", s3_data_100blk), ("PLC 100blk", plc_data_100blk)]
      data_list = [("S3 100blk", s3_data_100blk), ("PLC 100blk", plc_data_100blk), ("S3 nocache 100blk", s3_data_100blk_nocache), ("Syndicate 100blk", syndicate_data_100blk)]
      for (data_name, data) in data_list:
         if data.get(node) == None:
            print >> sys.stderr, "%s: no data for %s" % (node, data_name)
            valid = False
         elif data[node] == None:
            print >> sys.stderr, "%s: unparseable data" % (node, data_name)
            valid = False
         elif len(data[node]['exception']) > 0:
            print >> sys.stderr, "%s: exceptions on %s" % (node, data_name)
            valid = False

      if not valid:
         continue;

      """
      syndicate['first_1k'].append( syndicate_data_1k[node]['end_recv'][0] - syndicate_data_1k[node]['start_recv'][0] )
      syndicate['last_1k'].append( syndicate_data_1k[node]['end_recv'][-1] - syndicate_data_1k[node]['start_recv'][-1] )
      syndicate['first_1m'].append( syndicate_data_1M[node]['end_recv'][0] - syndicate_data_1M[node]['start_recv'][0] )
      syndicate['last_1m'].append( syndicate_data_1M[node]['end_recv'][-1] - syndicate_data_1M[node]['start_recv'][-1] )
      syndicate['first_50m'].append( syndicate_data_50M[node]['end_recv'][0] - syndicate_data_50M[node]['start_recv'][0] )
      syndicate['last_50m'].append( syndicate_data_50M[node]['end_recv'][-1] - syndicate_data_50M[node]['start_recv'][-1] )
      s3['first_20k'].append( s3_data_20k[node]['end_recv'][0] - s3_data_20k[node]['start_recv'][0] )
      s3['last_20k'].append( s3_data_20k[node]['end_recv'][-1] - s3_data_20k[node]['start_recv'][-1] )
      s3['first_50m'].append( s3_data_50M[node]['end_recv'][0] - s3_data_50M[node]['start_recv'][0] )
      s3['last_50m'].append( s3_data_50M[node]['end_recv'][-1] - s3_data_50M[node]['start_recv'][-1] )
      """
      s3['first_100blk'].append( s3_data_100blk[node]['end_recv'][0] - s3_data_100blk[node]['start_recv'][0])
      s3['last_100blk'].append( s3_data_100blk[node]['end_recv'][-1] - s3_data_100blk[node]['start_recv'][-1])

      s3['first_100blk_nocache'].append( s3_data_100blk_nocache[node]['end_recv'][0] - s3_data_100blk_nocache[node]['start_recv'][0] )
      plc['first_100blk'].append( plc_data_100blk[node]['end_recv'][0] - plc_data_100blk[node]['start_recv'][0])
      plc['last_100blk'].append( plc_data_100blk[node]['end_recv'][-1] - plc_data_100blk[node]['start_recv'][-1])
      syndicate['first_100blk'].append( syndicate_data_100blk[node]['end_recv'][0] - syndicate_data_100blk[node]['start_recv'][0] )
      syndicate['last_100blk'].append( syndicate_data_100blk[node]['end_recv'][-1] - syndicate_data_100blk[node]['start_recv'][-1] )

      if syndicate['first_100blk'][-1] > 150:
         slow.append( node )

      num_valid += 1

   #print "s3_first_100blk = " + str(s3['first_100blk'])
   #print "s3_last_100blk = " + str(s3['last_100blk'])

   print "valid: " + str(num_valid)
   print "slow: \n" + "\n".join(slow)

   # first 1K vs last 1K
   cdf_compare( [syndicate['first_100blk'], syndicate['last_100blk'], plc['first_100blk'] ], "Syndicate One-Writer-Many-Reader Download Times", "Seconds", np.arange(0, 1000, 100), "CDF(x)", np.arange(0, 1.05, 0.05), ["Syndicate 0% Cache Hit", "Syndicate 100% Cache Hit", "Python HTTP Server and Clients"] )

   #cdf_compare( [plc['first_100blk'], s3['first_100blk']], "Amazon S3 vs PLC Cache Miss Download Times", "Seconds", np.arange(0, 425, 30), "CDF(x)", np.arange(0, 1.05, 0.05) )
   cdf_compare( [s3['first_100blk'], s3['first_100blk_nocache']], "Amazon S3 Cache and Direct Download Times", "Seconds", np.arange(0, 1200, 100), "CDF(x)", np.arange(0, 1.05, 0.05), ["0% hit cache hit rate", "Direct Download"] )
   cdf_compare( [s3['first_100blk'], s3['last_100blk']], "Amazon S3 Cache Miss and Cache Hit Download Times", "Seconds", np.arange(0, 425, 30), "CDF(x)", np.arange(0, 1.05, 0.05) )

   cdf_compare( [syndicate['first_1k'], syndicate['last_1k']] )
   cdf_compare( [syndicate['first_50m'], s3['first_50m']] )
   cdf_compare( [syndicate['last_50m'], s3['last_50m']] )
   #cdf_compare( [syndicate['last_1m'], s3['last_20k']] )


# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PySncosmo(PythonPackage):
    """SNCosmo is a Python library for high-level supernova cosmology
    analysis."""

    homepage = "http://sncosmo.readthedocs.io/"
    url = "https://pypi.io/packages/source/s/sncosmo/sncosmo-1.2.0.tar.gz"

    version('1.2.0', sha256='f3969eec5b25f60c70418dbd64765a2b4735bb53c210c61d0aab68916daea588')

    # Required dependencies
    # py-sncosmo binaries are duplicates of those from py-astropy
    extends('python', ignore=r'bin/.*')
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy', type=('build', 'run'))
    depends_on('py-scipy', type=('build', 'run'))
    depends_on('py-astropy', type=('build', 'run'))

    # Recommended dependencies
    depends_on('py-matplotlib', type=('build', 'run'))
    depends_on('py-iminuit', type=('build', 'run'))
    depends_on('py-emcee', type=('build', 'run'))
    depends_on('py-nestle', type=('build', 'run'))


"""
This file is very long and growing, but it was decided to not split it yet, as
it's still manageable (2020-03-17, ~1.1k LoC). See gh-31989

Instead of splitting it was decided to define sections here:
- Configuration / Settings
- Autouse fixtures
- Common arguments
- Missing values & co.
- Classes
- Indices
- Series'
- DataFrames
- Operators & Operations
- Data sets/files
- Time zones
- Dtypes
- Misc
"""

from collections import abc
from datetime import date, time, timedelta, timezone
from decimal import Decimal
import operator
import os

from dateutil.tz import tzlocal, tzutc
import hypothesis
from hypothesis import strategies as st
import numpy as np
import pytest
from pytz import FixedOffset, utc

import pandas.util._test_decorators as td

from pandas.core.dtypes.dtypes import DatetimeTZDtype, IntervalDtype

import pandas as pd
from pandas import DataFrame, Interval, Period, Series, Timedelta, Timestamp
import pandas._testing as tm
from pandas.core import ops
from pandas.core.indexes.api import Index, MultiIndex

# ----------------------------------------------------------------
# Configuration / Settings
# ----------------------------------------------------------------
# pytest
def pytest_configure(config):
    # Register marks to avoid warnings in pandas.test()
    # sync with setup.cfg
    config.addinivalue_line("markers", "single: mark a test as single cpu only")
    config.addinivalue_line("markers", "slow: mark a test as slow")
    config.addinivalue_line("markers", "network: mark a test as network")
    config.addinivalue_line(
        "markers", "db: tests requiring a database (mysql or postgres)"
    )
    config.addinivalue_line("markers", "high_memory: mark a test as a high-memory only")
    config.addinivalue_line("markers", "clipboard: mark a pd.read_clipboard test")
    config.addinivalue_line(
        "markers", "arm_slow: mark a test as slow for arm64 architecture"
    )

def pytest_addoption(parser):
    parser.addoption("--skip-slow", action="store_true", help="skip slow tests")
    parser.addoption("--skip-network", action="store_true", help="skip network tests")
    parser.addoption("--skip-db", action="store_true", help="skip db tests")
    parser.addoption(
        "--run-high-memory", action="store_true", help="run high memory tests"
    )
    parser.addoption("--only-slow", action="store_true", help="run only slow tests")
    parser.addoption(
        "--strict-data-files",
        action="store_true",
        help="Fail if a test is skipped for missing data file.",
    )

def pytest_runtest_setup(item):
    if "slow" in item.keywords and item.config.getoption("--skip-slow"):
        pytest.skip("skipping due to --skip-slow")

    if "slow" not in item.keywords and item.config.getoption("--only-slow"):
        pytest.skip("skipping due to --only-slow")

    if "network" in item.keywords and item.config.getoption("--skip-network"):
        pytest.skip("skipping due to --skip-network")

    if "db" in item.keywords and item.config.getoption("--skip-db"):
        pytest.skip("skipping due to --skip-db")

    if "high_memory" in item.keywords and not item.config.getoption(
        "--run-high-memory"
    ):
        pytest.skip("skipping high memory test since --run-high-memory was not set")

# Hypothesis
hypothesis.settings.register_profile(
    "ci",
    # Hypothesis timing checks are tuned for scalars by default, so we bump
    # them from 200ms to 500ms per test case as the global default.  If this
    # is too short for a specific test, (a) try to make it faster, and (b)
    # if it really is slow add `@settings(deadline=...)` with a working value,
    # or `deadline=None` to entirely disable timeouts for that test.
    deadline=500,
    suppress_health_check=(hypothesis.HealthCheck.too_slow,),
)
hypothesis.settings.load_profile("ci")

# Registering these strategies makes them globally available via st.from_type,
# which is use for offsets in tests/tseries/offsets/test_offsets_properties.py
for name in "MonthBegin MonthEnd BMonthBegin BMonthEnd".split():
    cls = getattr(pd.tseries.offsets, name)
    st.register_type_strategy(
        cls, st.builds(cls, n=st.integers(-99, 99), normalize=st.booleans())
    )

for name in "YearBegin YearEnd BYearBegin BYearEnd".split():
    cls = getattr(pd.tseries.offsets, name)
    st.register_type_strategy(
        cls,
        st.builds(
            cls,
            n=st.integers(-5, 5),
            normalize=st.booleans(),
            month=st.integers(min_value=1, max_value=12),
        ),
    )

for name in "QuarterBegin QuarterEnd BQuarterBegin BQuarterEnd".split():
    cls = getattr(pd.tseries.offsets, name)
    st.register_type_strategy(
        cls,
        st.builds(
            cls,
            n=st.integers(-24, 24),
            normalize=st.booleans(),
            startingMonth=st.integers(min_value=1, max_value=12),
        ),
    )

# ----------------------------------------------------------------
# Autouse fixtures
# ----------------------------------------------------------------
@pytest.fixture(autouse=True)
def configure_tests():
    """
    Configure settings for all tests and test modules.
    """
    pd.set_option("chained_assignment", "raise")

@pytest.fixture(autouse=True)
def add_imports(doctest_namespace):
    """
    Make `np` and `pd` names available for doctests.
    """
    doctest_namespace["np"] = np
    doctest_namespace["pd"] = pd

# ----------------------------------------------------------------
# Common arguments
# ----------------------------------------------------------------
@pytest.fixture(params=[0, 1, "index", "columns"], ids=lambda x: f"axis {repr(x)}")
def axis(request):
    """
    Fixture for returning the axis numbers of a DataFrame.
    """
    return request.param

axis_frame = axis

@pytest.fixture(params=[True, False, None])
def observed(request):
    """
    Pass in the observed keyword to groupby for [True, False]
    This indicates whether categoricals should return values for
    values which are not in the grouper [False / None], or only values which
    appear in the grouper [True]. [None] is supported for future compatibility
    if we decide to change the default (and would need to warn if this
    parameter is not passed).
    """
    return request.param

@pytest.fixture(params=[True, False, None])
def ordered(request):
    """
    Boolean 'ordered' parameter for Categorical.
    """
    return request.param

@pytest.fixture(params=["first", "last", False])
def keep(request):
    """
    Valid values for the 'keep' parameter used in
    .duplicated or .drop_duplicates
    """
    return request.param

@pytest.fixture(params=["left", "right", "both", "neither"])
def closed(request):
    """
    Fixture for trying all interval closed parameters.
    """
    return request.param

@pytest.fixture(params=["left", "right", "both", "neither"])
def other_closed(request):
    """
    Secondary closed fixture to allow parametrizing over all pairs of closed.
    """
    return request.param

@pytest.fixture(params=[None, "gzip", "bz2", "zip", "xz"])
def compression(request):
    """
    Fixture for trying common compression types in compression tests.
    """
    return request.param

@pytest.fixture(params=["gzip", "bz2", "zip", "xz"])
def compression_only(request):
    """
    Fixture for trying common compression types in compression tests excluding
    uncompressed case.
    """
    return request.param

@pytest.fixture(params=[True, False])
def writable(request):
    """
    Fixture that an array is writable.
    """
    return request.param

@pytest.fixture(params=["inner", "outer", "left", "right"])
def join_type(request):
    """
    Fixture for trying all types of join operations.
    """
    return request.param

@pytest.fixture(params=["nlargest", "nsmallest"])
def nselect_method(request):
    """
    Fixture for trying all nselect methods.
    """
    return request.param

# ----------------------------------------------------------------
# Missing values & co.
# ----------------------------------------------------------------
@pytest.fixture(params=tm.NULL_OBJECTS, ids=str)
def nulls_fixture(request):
    """
    Fixture for each null type in pandas.
    """
    return request.param

nulls_fixture2 = nulls_fixture  # Generate cartesian product of nulls_fixture

@pytest.fixture(params=[None, np.nan, pd.NaT])
def unique_nulls_fixture(request):
    """
    Fixture for each null type in pandas, each null type exactly once.
    """
    return request.param

# Generate cartesian product of unique_nulls_fixture:
unique_nulls_fixture2 = unique_nulls_fixture

# ----------------------------------------------------------------
# Classes
# ----------------------------------------------------------------

@pytest.fixture(params=[pd.DataFrame, pd.Series])
def frame_or_series(request):
    """
    Fixture to parametrize over DataFrame and Series.
    """
    return request.param

@pytest.fixture(
    params=[pd.Index, pd.Series], ids=["index", "series"]  # type: ignore[list-item]
)
def index_or_series(request):
    """
    Fixture to parametrize over Index and Series, made necessary by a mypy
    bug, giving an error:

    List item 0 has incompatible type "Type[Series]"; expected "Type[PandasObject]"

    See GH#29725
    """
    return request.param

# Generate cartesian product of index_or_series fixture:
index_or_series2 = index_or_series

@pytest.fixture(
    params=[pd.Index, pd.Series, pd.array], ids=["index", "series", "array"]
)
def index_or_series_or_array(request):
    """
    Fixture to parametrize over Index, Series, and ExtensionArray
    """
    return request.param

@pytest.fixture
def dict_subclass():
    """
    Fixture for a dictionary subclass.
    """

    class TestSubDict(dict):
        def __init__(self, *args, **kwargs):
            dict.__init__(self, *args, **kwargs)

    return TestSubDict

@pytest.fixture
def non_dict_mapping_subclass():
    """
    Fixture for a non-mapping dictionary subclass.
    """

    class TestNonDictMapping(abc.Mapping):
        def __init__(self, underlying_dict):
            self._data = underlying_dict

        def __getitem__(self, key):
            return self._data.__getitem__(key)

        def __iter__(self):
            return self._data.__iter__()

        def __len__(self):
            return self._data.__len__()

    return TestNonDictMapping

# ----------------------------------------------------------------
# Indices
# ----------------------------------------------------------------
@pytest.fixture
def multiindex_year_month_day_dataframe_random_data():
    """
    DataFrame with 3 level MultiIndex (year, month, day) covering
    first 100 business days from 2000-01-01 with random data
    """
    tdf = tm.makeTimeDataFrame(100)
    ymd = tdf.groupby([lambda x: x.year, lambda x: x.month, lambda x: x.day]).sum()
    # use Int64Index, to make sure things work
    ymd.index = ymd.index.set_levels([lev.astype("i8") for lev in ymd.index.levels])
    ymd.index.set_names(["year", "month", "day"], inplace=True)
    return ymd

@pytest.fixture
def multiindex_dataframe_random_data():
    """DataFrame with 2 level MultiIndex with random data"""
    index = MultiIndex(
        levels=[["foo", "bar", "baz", "qux"], ["one", "two", "three"]],
        codes=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3], [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
        names=["first", "second"],
    )
    return DataFrame(
        np.random.randn(10, 3), index=index, columns=Index(["A", "B", "C"], name="exp")
    )

def _create_multiindex():
    """
    MultiIndex used to test the general functionality of this object
    """

    # See Also: tests.multi.conftest.idx
    major_axis = Index(["foo", "bar", "baz", "qux"])
    minor_axis = Index(["one", "two"])

    major_codes = np.array([0, 0, 1, 2, 3, 3])
    minor_codes = np.array([0, 1, 0, 1, 0, 1])
    index_names = ["first", "second"]
    return MultiIndex(
        levels=[major_axis, minor_axis],
        codes=[major_codes, minor_codes],
        names=index_names,
        verify_integrity=False,
    )

def _create_mi_with_dt64tz_level():
    """
    MultiIndex with a level that is a tzaware DatetimeIndex.
    """
    # GH#8367 round trip with pickle
    return MultiIndex.from_product(
        [[1, 2], ["a", "b"], pd.date_range("20130101", periods=3, tz="US/Eastern")],
        names=["one", "two", "three"],
    )

indices_dict = {
    "unicode": tm.makeUnicodeIndex(100),
    "string": tm.makeStringIndex(100),
    "datetime": tm.makeDateIndex(100),
    "datetime-tz": tm.makeDateIndex(100, tz="US/Pacific"),
    "period": tm.makePeriodIndex(100),
    "timedelta": tm.makeTimedeltaIndex(100),
    "int": tm.makeIntIndex(100),
    "uint": tm.makeUIntIndex(100),
    "range": tm.makeRangeIndex(100),
    "float": tm.makeFloatIndex(100),
    "bool": tm.makeBoolIndex(10),
    "categorical": tm.makeCategoricalIndex(100),
    "interval": tm.makeIntervalIndex(100),
    "empty": Index([]),
    "tuples": MultiIndex.from_tuples(zip(["foo", "bar", "baz"], [1, 2, 3])),
    "mi-with-dt64tz-level": _create_mi_with_dt64tz_level(),
    "multi": _create_multiindex(),
    "repeats": Index([0, 0, 1, 1, 2, 2]),
}

@pytest.fixture(params=indices_dict.keys())
def index(request):
    """
    Fixture for many "simple" kinds of indices.

    These indices are unlikely to cover corner cases, e.g.
        - no names
        - no NaTs/NaNs
        - no values near implementation bounds
        - ...
    """
    # copy to avoid mutation, e.g. setting .name
    return indices_dict[request.param].copy()

# Needed to generate cartesian product of indices
index_fixture2 = index

@pytest.fixture(params=indices_dict.keys())
def index_with_missing(request):
    """
    Fixture for indices with missing values
    """
    if request.param in ["int", "uint", "range", "empty", "repeats"]:
        pytest.xfail("missing values not supported")
    # GH 35538. Use deep copy to avoid illusive bug on np-dev
    # Azure pipeline that writes into indices_dict despite copy
    ind = indices_dict[request.param].copy(deep=True)
    vals = ind.values
    if request.param in ["tuples", "mi-with-dt64tz-level", "multi"]:
        # For setting missing values in the top level of MultiIndex
        vals = ind.tolist()
        vals[0] = (None,) + vals[0][1:]
        vals[-1] = (None,) + vals[-1][1:]
        return MultiIndex.from_tuples(vals)
    else:
        vals[0] = None
        vals[-1] = None
        return type(ind)(vals)

# ----------------------------------------------------------------
# Series'
# ----------------------------------------------------------------
@pytest.fixture
def empty_series():
    return pd.Series([], index=[], dtype=np.float64)

@pytest.fixture
def string_series():
    """
    Fixture for Series of floats with Index of unique strings
    """
    s = tm.makeStringSeries()
    s.name = "series"
    return s

@pytest.fixture
def object_series():
    """
    Fixture for Series of dtype object with Index of unique strings
    """
    s = tm.makeObjectSeries()
    s.name = "objects"
    return s

@pytest.fixture
def datetime_series():
    """
    Fixture for Series of floats with DatetimeIndex
    """
    s = tm.makeTimeSeries()
    s.name = "ts"
    return s

def _create_series(index):
    """ Helper for the _series dict """
    size = len(index)
    data = np.random.randn(size)
    return pd.Series(data, index=index, name="a")

_series = {
    f"series-with-{index_id}-index": _create_series(index)
    for index_id, index in indices_dict.items()
}

@pytest.fixture
def series_with_simple_index(index):
    """
    Fixture for tests on series with changing types of indices.
    """
    return _create_series(index)

@pytest.fixture
def series_with_multilevel_index():
    """
    Fixture with a Series with a 2-level MultiIndex.
    """
    arrays = [
        ["bar", "bar", "baz", "baz", "qux", "qux", "foo", "foo"],
        ["one", "two", "one", "two", "one", "two", "one", "two"],
    ]
    tuples = zip(*arrays)
    index = MultiIndex.from_tuples(tuples)
    data = np.random.randn(8)
    ser = Series(data, index=index)
    ser[3] = np.NaN
    return ser

_narrow_dtypes = [
    np.float16,
    np.float32,
    np.int8,
    np.int16,
    np.int32,
    np.uint8,
    np.uint16,
    np.uint32,
]
_narrow_series = {
    f"{dtype.__name__}-series": tm.makeFloatSeries(name="a").astype(dtype)
    for dtype in _narrow_dtypes
}

@pytest.fixture(params=_narrow_series.keys())
def narrow_series(request):
    """
    Fixture for Series with low precision data types
    """
    # copy to avoid mutation, e.g. setting .name
    return _narrow_series[request.param].copy()

_index_or_series_objs = {**indices_dict, **_series, **_narrow_series}

@pytest.fixture(params=_index_or_series_objs.keys())
def index_or_series_obj(request):
    """
    Fixture for tests on indexes, series and series with a narrow dtype
    copy to avoid mutation, e.g. setting .name
    """
    return _index_or_series_objs[request.param].copy(deep=True)

# ----------------------------------------------------------------
# DataFrames
# ----------------------------------------------------------------
@pytest.fixture
def empty_frame():
    return DataFrame()

@pytest.fixture
def int_frame():
    """
    Fixture for DataFrame of ints with index of unique strings

    Columns are ['A', 'B', 'C', 'D']

                A  B  C  D
    vpBeWjM651  1  0  1  0
    5JyxmrP1En -1  0  0  0
    qEDaoD49U2 -1  1  0  0
    m66TkTfsFe  0  0  0  0
    EHPaNzEUFm -1  0 -1  0
    fpRJCevQhi  2  0  0  0
    OlQvnmfi3Q  0  0 -2  0
    ...        .. .. .. ..
    uB1FPlz4uP  0  0  0  1
    EcSe6yNzCU  0  0 -1  0
    L50VudaiI8 -1  1 -2  0
    y3bpw4nwIp  0 -1  0  0
    H0RdLLwrCT  1  1  0  0
    rY82K0vMwm  0  0  0  0
    1OPIUjnkjk  2  0  0  0

    [30 rows x 4 columns]
    """
    return DataFrame(tm.getSeriesData()).astype("int64")

@pytest.fixture
def datetime_frame():
    """
    Fixture for DataFrame of floats with DatetimeIndex

    Columns are ['A', 'B', 'C', 'D']

                       A         B         C         D
    2000-01-03 -1.122153  0.468535  0.122226  1.693711
    2000-01-04  0.189378  0.486100  0.007864 -1.216052
    2000-01-05  0.041401 -0.835752 -0.035279 -0.414357
    2000-01-06  0.430050  0.894352  0.090719  0.036939
    2000-01-07 -0.620982 -0.668211 -0.706153  1.466335
    2000-01-10 -0.752633  0.328434 -0.815325  0.699674
    2000-01-11 -2.236969  0.615737 -0.829076 -1.196106
    ...              ...       ...       ...       ...
    2000-02-03  1.642618 -0.579288  0.046005  1.385249
    2000-02-04 -0.544873 -1.160962 -0.284071 -1.418351
    2000-02-07 -2.656149 -0.601387  1.410148  0.444150
    2000-02-08 -1.201881 -1.289040  0.772992 -1.445300
    2000-02-09  1.377373  0.398619  1.008453 -0.928207
    2000-02-10  0.473194 -0.636677  0.984058  0.511519
    2000-02-11 -0.965556  0.408313 -1.312844 -0.381948

    [30 rows x 4 columns]
    """
    return DataFrame(tm.getTimeSeriesData())

@pytest.fixture
def float_frame():
    """
    Fixture for DataFrame of floats with index of unique strings

    Columns are ['A', 'B', 'C', 'D'].

                       A         B         C         D
    P7GACiRnxd -0.465578 -0.361863  0.886172 -0.053465
    qZKh6afn8n -0.466693 -0.373773  0.266873  1.673901
    tkp0r6Qble  0.148691 -0.059051  0.174817  1.598433
    wP70WOCtv8  0.133045 -0.581994 -0.992240  0.261651
    M2AeYQMnCz -1.207959 -0.185775  0.588206  0.563938
    QEPzyGDYDo -0.381843 -0.758281  0.502575 -0.565053
    r78Jwns6dn -0.653707  0.883127  0.682199  0.206159
    ...              ...       ...       ...       ...
    IHEGx9NO0T -0.277360  0.113021 -1.018314  0.196316
    lPMj8K27FA -1.313667 -0.604776 -1.305618 -0.863999
    qa66YMWQa5  1.110525  0.475310 -0.747865  0.032121
    yOa0ATsmcE -0.431457  0.067094  0.096567 -0.264962
    65znX3uRNG  1.528446  0.160416 -0.109635 -0.032987
    eCOBvKqf3e  0.235281  1.622222  0.781255  0.392871
    xSucinXxuV -1.263557  0.252799 -0.552247  0.400426

    [30 rows x 4 columns]
    """
    return DataFrame(tm.getSeriesData())

# ----------------------------------------------------------------
# Scalars
# ----------------------------------------------------------------
@pytest.fixture(
    params=[
        (Interval(left=0, right=5), IntervalDtype("int64")),
        (Interval(left=0.1, right=0.5), IntervalDtype("float64")),
        (Period("2012-01", freq="M"), "period[M]"),
        (Period("2012-02-01", freq="D"), "period[D]"),
        (
            Timestamp("2011-01-01", tz="US/Eastern"),
            DatetimeTZDtype(tz="US/Eastern"),
        ),
        (Timedelta(seconds=500), "timedelta64[ns]"),
    ]
)
def ea_scalar_and_dtype(request):
    return request.param

# ----------------------------------------------------------------
# Operators & Operations
# ----------------------------------------------------------------
_all_arithmetic_operators = [
    "__add__",
    "__radd__",
    "__sub__",
    "__rsub__",
    "__mul__",
    "__rmul__",
    "__floordiv__",
    "__rfloordiv__",
    "__truediv__",
    "__rtruediv__",
    "__pow__",
    "__rpow__",
    "__mod__",
    "__rmod__",
]

@pytest.fixture(params=_all_arithmetic_operators)
def all_arithmetic_operators(request):
    """
    Fixture for dunder names for common arithmetic operations.
    """
    return request.param

@pytest.fixture(
    params=[
        operator.add,
        ops.radd,
        operator.sub,
        ops.rsub,
        operator.mul,
        ops.rmul,
        operator.truediv,
        ops.rtruediv,
        operator.floordiv,
        ops.rfloordiv,
        operator.mod,
        ops.rmod,
        operator.pow,
        ops.rpow,
        operator.eq,
        operator.ne,
        operator.lt,
        operator.le,
        operator.gt,
        operator.ge,
        operator.and_,
        ops.rand_,
        operator.xor,
        ops.rxor,
        operator.or_,
        ops.ror_,
    ]
)
def all_binary_operators(request):
    """
    Fixture for operator and roperator arithmetic, comparison, and logical ops.
    """
    return request.param

@pytest.fixture(
    params=[
        operator.add,
        ops.radd,
        operator.sub,
        ops.rsub,
        operator.mul,
        ops.rmul,
        operator.truediv,
        ops.rtruediv,
        operator.floordiv,
        ops.rfloordiv,
        operator.mod,
        ops.rmod,
        operator.pow,
        ops.rpow,
    ]
)
def all_arithmetic_functions(request):
    """
    Fixture for operator and roperator arithmetic functions.

    Notes
    -----
    This includes divmod and rdivmod, whereas all_arithmetic_operators
    does not.
    """
    return request.param

_all_numeric_reductions = [
    "sum",
    "max",
    "min",
    "mean",
    "prod",
    "std",
    "var",
    "median",
    "kurt",
    "skew",
]

@pytest.fixture(params=_all_numeric_reductions)
def all_numeric_reductions(request):
    """
    Fixture for numeric reduction names.
    """
    return request.param

_all_boolean_reductions = ["all", "any"]

@pytest.fixture(params=_all_boolean_reductions)
def all_boolean_reductions(request):
    """
    Fixture for boolean reduction names.
    """
    return request.param

_all_reductions = _all_numeric_reductions + _all_boolean_reductions

@pytest.fixture(params=_all_reductions)
def all_reductions(request):
    """
    Fixture for all (boolean + numeric) reduction names.
    """
    return request.param

@pytest.fixture(params=["__eq__", "__ne__", "__le__", "__lt__", "__ge__", "__gt__"])
def all_compare_operators(request):
    """
    Fixture for dunder names for common compare operations

    * >=
    * >
    * ==
    * !=
    * <
    * <=
    """
    return request.param

@pytest.fixture(params=["__le__", "__lt__", "__ge__", "__gt__"])
def compare_operators_no_eq_ne(request):
    """
    Fixture for dunder names for compare operations except == and !=

    * >=
    * >
    * <
    * <=
    """
    return request.param

@pytest.fixture(
    params=["__and__", "__rand__", "__or__", "__ror__", "__xor__", "__rxor__"]
)
def all_logical_operators(request):
    """
    Fixture for dunder names for common logical operations

    * |
    * &
    * ^
    """
    return request.param

# ----------------------------------------------------------------
# Data sets/files
# ----------------------------------------------------------------
@pytest.fixture
def strict_data_files(pytestconfig):
    """
    Returns the configuration for the test setting `--strict-data-files`.
    """
    return pytestconfig.getoption("--strict-data-files")

@pytest.fixture
def datapath(strict_data_files):
    """
    Get the path to a data file.

    Parameters
    ----------
    path : str
        Path to the file, relative to ``pandas/tests/``

    Returns
    -------
    path including ``pandas/tests``.

    Raises
    ------
    ValueError
        If the path doesn't exist and the --strict-data-files option is set.
    """
    BASE_PATH = os.path.join(os.path.dirname(__file__), "tests")

    def deco(*args):
        path = os.path.join(BASE_PATH, *args)
        if not os.path.exists(path):
            if strict_data_files:
                raise ValueError(
                    f"Could not find file {path} and --strict-data-files is set."
                )
            else:
                pytest.skip(f"Could not find {path}.")
        return path

    return deco

@pytest.fixture
def iris(datapath):
    """
    The iris dataset as a DataFrame.
    """
    return pd.read_csv(datapath("io", "data", "csv", "iris.csv"))

# ----------------------------------------------------------------
# Time zones
# ----------------------------------------------------------------
TIMEZONES = [
    None,
    "UTC",
    "US/Eastern",
    "Asia/Tokyo",
    "dateutil/US/Pacific",
    "dateutil/Asia/Singapore",
    "+01:15",
    "-02:15",
    "UTC+01:15",
    "UTC-02:15",
    tzutc(),
    tzlocal(),
    FixedOffset(300),
    FixedOffset(0),
    FixedOffset(-300),
    timezone.utc,
    timezone(timedelta(hours=1)),
    timezone(timedelta(hours=-1), name="foo"),
]
TIMEZONE_IDS = [repr(i) for i in TIMEZONES]

@td.parametrize_fixture_doc(str(TIMEZONE_IDS))
@pytest.fixture(params=TIMEZONES, ids=TIMEZONE_IDS)
def tz_naive_fixture(request):
    """
    Fixture for trying timezones including default (None): {0}
    """
    return request.param

@td.parametrize_fixture_doc(str(TIMEZONE_IDS[1:]))
@pytest.fixture(params=TIMEZONES[1:], ids=TIMEZONE_IDS[1:])
def tz_aware_fixture(request):
    """
    Fixture for trying explicit timezones: {0}
    """
    return request.param

# Generate cartesian product of tz_aware_fixture:
tz_aware_fixture2 = tz_aware_fixture

@pytest.fixture(scope="module")
def datetime_tz_utc():
    """
    Yields the UTC timezone object from the datetime module.
    """
    return timezone.utc

@pytest.fixture(params=["utc", "dateutil/UTC", utc, tzutc(), timezone.utc])
def utc_fixture(request):
    """
    Fixture to provide variants of UTC timezone strings and tzinfo objects.
    """
    return request.param

# ----------------------------------------------------------------
# Dtypes
# ----------------------------------------------------------------
@pytest.fixture(params=tm.STRING_DTYPES)
def string_dtype(request):
    """
    Parametrized fixture for string dtypes.

    * str
    * 'str'
    * 'U'
    """
    return request.param

@pytest.fixture(params=tm.BYTES_DTYPES)
def bytes_dtype(request):
    """
    Parametrized fixture for bytes dtypes.

    * bytes
    * 'bytes'
    """
    return request.param

@pytest.fixture(params=tm.OBJECT_DTYPES)
def object_dtype(request):
    """
    Parametrized fixture for object dtypes.

    * object
    * 'object'
    """
    return request.param

@pytest.fixture(params=tm.DATETIME64_DTYPES)
def datetime64_dtype(request):
    """
    Parametrized fixture for datetime64 dtypes.

    * 'datetime64[ns]'
    * 'M8[ns]'
    """
    return request.param

@pytest.fixture(params=tm.TIMEDELTA64_DTYPES)
def timedelta64_dtype(request):
    """
    Parametrized fixture for timedelta64 dtypes.

    * 'timedelta64[ns]'
    * 'm8[ns]'
    """
    return request.param

@pytest.fixture(params=tm.FLOAT_DTYPES)
def float_dtype(request):
    """
    Parameterized fixture for float dtypes.

    * float
    * 'float32'
    * 'float64'
    """
    return request.param

@pytest.fixture(params=tm.FLOAT_EA_DTYPES)
def float_ea_dtype(request):
    """
    Parameterized fixture for float dtypes.

    * 'Float32'
    * 'Float64'
    """
    return request.param

@pytest.fixture(params=tm.FLOAT_DTYPES + tm.FLOAT_EA_DTYPES)
def any_float_allowed_nullable_dtype(request):
    """
    Parameterized fixture for float dtypes.

    * float
    * 'float32'
    * 'float64'
    * 'Float32'
    * 'Float64'
    """
    return request.param

@pytest.fixture(params=tm.COMPLEX_DTYPES)
def complex_dtype(request):
    """
    Parameterized fixture for complex dtypes.

    * complex
    * 'complex64'
    * 'complex128'
    """
    return request.param

@pytest.fixture(params=tm.SIGNED_INT_DTYPES)
def sint_dtype(request):
    """
    Parameterized fixture for signed integer dtypes.

    * int
    * 'int8'
    * 'int16'
    * 'int32'
    * 'int64'
    """
    return request.param

@pytest.fixture(params=tm.UNSIGNED_INT_DTYPES)
def uint_dtype(request):
    """
    Parameterized fixture for unsigned integer dtypes.

    * 'uint8'
    * 'uint16'
    * 'uint32'
    * 'uint64'
    """
    return request.param

@pytest.fixture(params=tm.ALL_INT_DTYPES)
def any_int_dtype(request):
    """
    Parameterized fixture for any integer dtype.

    * int
    * 'int8'
    * 'uint8'
    * 'int16'
    * 'uint16'
    * 'int32'
    * 'uint32'
    * 'int64'
    * 'uint64'
    """
    return request.param

@pytest.fixture(params=tm.ALL_EA_INT_DTYPES)
def any_nullable_int_dtype(request):
    """
    Parameterized fixture for any nullable integer dtype.

    * 'UInt8'
    * 'Int8'
    * 'UInt16'
    * 'Int16'
    * 'UInt32'
    * 'Int32'
    * 'UInt64'
    * 'Int64'
    """
    return request.param

@pytest.fixture(params=tm.ALL_EA_INT_DTYPES + tm.FLOAT_EA_DTYPES)
def any_numeric_dtype(request):
    """
    Parameterized fixture for any nullable integer dtype and
    any float ea dtypes.

    * 'UInt8'
    * 'Int8'
    * 'UInt16'
    * 'Int16'
    * 'UInt32'
    * 'Int32'
    * 'UInt64'
    * 'Int64'
    * 'Float32'
    * 'Float64'
    """
    return request.param

@pytest.fixture(params=tm.SIGNED_EA_INT_DTYPES)
def any_signed_nullable_int_dtype(request):
    """
    Parameterized fixture for any signed nullable integer dtype.

    * 'Int8'
    * 'Int16'
    * 'Int32'
    * 'Int64'
    """
    return request.param

@pytest.fixture(params=tm.ALL_REAL_DTYPES)
def any_real_dtype(request):
    """
    Parameterized fixture for any (purely) real numeric dtype.

    * int
    * 'int8'
    * 'uint8'
    * 'int16'
    * 'uint16'
    * 'int32'
    * 'uint32'
    * 'int64'
    * 'uint64'
    * float
    * 'float32'
    * 'float64'
    """
    return request.param

@pytest.fixture(params=tm.ALL_NUMPY_DTYPES)
def any_numpy_dtype(request):
    """
    Parameterized fixture for all numpy dtypes.

    * bool
    * 'bool'
    * int
    * 'int8'
    * 'uint8'
    * 'int16'
    * 'uint16'
    * 'int32'
    * 'uint32'
    * 'int64'
    * 'uint64'
    * float
    * 'float32'
    * 'float64'
    * complex
    * 'complex64'
    * 'complex128'
    * str
    * 'str'
    * 'U'
    * bytes
    * 'bytes'
    * 'datetime64[ns]'
    * 'M8[ns]'
    * 'timedelta64[ns]'
    * 'm8[ns]'
    * object
    * 'object'
    """
    return request.param

# categoricals are handled separately
_any_skipna_inferred_dtype = [
    ("string", ["a", np.nan, "c"]),
    ("string", ["a", pd.NA, "c"]),
    ("bytes", [b"a", np.nan, b"c"]),
    ("empty", [np.nan, np.nan, np.nan]),
    ("empty", []),
    ("mixed-integer", ["a", np.nan, 2]),
    ("mixed", ["a", np.nan, 2.0]),
    ("floating", [1.0, np.nan, 2.0]),
    ("integer", [1, np.nan, 2]),
    ("mixed-integer-float", [1, np.nan, 2.0]),
    ("decimal", [Decimal(1), np.nan, Decimal(2)]),
    ("boolean", [True, np.nan, False]),
    ("boolean", [True, pd.NA, False]),
    ("datetime64", [np.datetime64("2013-01-01"), np.nan, np.datetime64("2018-01-01")]),
    ("datetime", [pd.Timestamp("20130101"), np.nan, pd.Timestamp("20180101")]),
    ("date", [date(2013, 1, 1), np.nan, date(2018, 1, 1)]),
    # The following two dtypes are commented out due to GH 23554
    # ('complex', [1 + 1j, np.nan, 2 + 2j]),
    # ('timedelta64', [np.timedelta64(1, 'D'),
    #                  np.nan, np.timedelta64(2, 'D')]),
    ("timedelta", [timedelta(1), np.nan, timedelta(2)]),
    ("time", [time(1), np.nan, time(2)]),
    ("period", [pd.Period(2013), pd.NaT, pd.Period(2018)]),
    ("interval", [pd.Interval(0, 1), np.nan, pd.Interval(0, 2)]),
]
ids, _ = zip(*_any_skipna_inferred_dtype)  # use inferred type as fixture-id

@pytest.fixture(params=_any_skipna_inferred_dtype, ids=ids)
def any_skipna_inferred_dtype(request):
    """
    Fixture for all inferred dtypes from _libs.lib.infer_dtype

    The covered (inferred) types are:
    * 'string'
    * 'empty'
    * 'bytes'
    * 'mixed'
    * 'mixed-integer'
    * 'mixed-integer-float'
    * 'floating'
    * 'integer'
    * 'decimal'
    * 'boolean'
    * 'datetime64'
    * 'datetime'
    * 'date'
    * 'timedelta'
    * 'time'
    * 'period'
    * 'interval'

    Returns
    -------
    inferred_dtype : str
        The string for the inferred dtype from _libs.lib.infer_dtype
    values : np.ndarray
        An array of object dtype that will be inferred to have
        `inferred_dtype`

    Examples
    --------
    >>> import pandas._libs.lib as lib
    >>>
    >>> def test_something(any_skipna_inferred_dtype):
    ...     inferred_dtype, values = any_skipna_inferred_dtype
    ...     # will pass
    ...     assert lib.infer_dtype(values, skipna=True) == inferred_dtype
    """
    inferred_dtype, values = request.param
    values = np.array(values, dtype=object)  # object dtype to avoid casting

    # correctness of inference tested in tests/dtypes/test_inference.py
    return inferred_dtype, values

# ----------------------------------------------------------------
# Misc
# ----------------------------------------------------------------
@pytest.fixture
def ip():
    """
    Get an instance of IPython.InteractiveShell.

    Will raise a skip if IPython is not installed.
    """
    pytest.importorskip("IPython", minversion="6.0.0")
    from IPython.core.interactiveshell import InteractiveShell

    # GH#35711 make sure sqlite history file handle is not leaked
    from traitlets.config import Config  # isort:skip

    c = Config()
    c.HistoryManager.hist_file = ":memory:"

    return InteractiveShell(config=c)

@pytest.fixture(params=["bsr", "coo", "csc", "csr", "dia", "dok", "lil"])
def spmatrix(request):
    """
    Yields scipy sparse matrix classes.
    """
    from scipy import sparse

    return getattr(sparse, request.param + "_matrix")

@pytest.fixture(
    params=[
        getattr(pd.offsets, o)
        for o in pd.offsets.__all__
        if issubclass(getattr(pd.offsets, o), pd.offsets.Tick)
    ]
)
def tick_classes(request):
    """
    Fixture for Tick based datetime offsets available for a time series.
    """
    return request.param

@pytest.fixture(params=[None, lambda x: x])
def sort_by_key(request):
    """
    Simple fixture for testing keys in sorting methods.
    Tests None (no key) and the identity key.
    """
    return request.param

@pytest.fixture()
def fsspectest():
    pytest.importorskip("fsspec")
    from fsspec import register_implementation
    from fsspec.implementations.memory import MemoryFileSystem
    from fsspec.registry import _registry as registry

    class TestMemoryFS(MemoryFileSystem):
        protocol = "testmem"
        test = [None]

        def __init__(self, **kwargs):
            self.test[0] = kwargs.pop("test", None)
            super().__init__(**kwargs)

    register_implementation("testmem", TestMemoryFS, clobber=True)
    yield TestMemoryFS()
    registry.pop("testmem", None)
    TestMemoryFS.test[0] = None
    TestMemoryFS.store.clear()

@pytest.fixture(
    params=[
        ("foo", None, None),
        ("Egon", "Venkman", None),
        ("NCC1701D", "NCC1701D", "NCC1701D"),
    ]
)
def names(request):
    """
    A 3-tuple of names, the first two for operands, the last for a result.
    """
    return request.param

@pytest.fixture(params=[tm.setitem, tm.loc, tm.iloc])
def indexer_sli(request):
    """
    Parametrize over __setitem__, loc.__setitem__, iloc.__setitem__
    """
    return request.param

@pytest.fixture(params=[tm.setitem, tm.iloc])
def indexer_si(request):
    """
    Parametrize over __setitem__, iloc.__setitem__
    """
    return request.param


# Author: Hamzeh Alsalhi <ha258@cornell.edu>
#
# License: BSD 3 clause
from __future__ import division
import numpy as np
import scipy.sparse as sp
import operator
import array

from sklearn.utils import check_random_state
from sklearn.utils.fixes import astype
from ._random import sample_without_replacement

__all__ = ['sample_without_replacement', 'choice']

# This is a backport of np.random.choice from numpy 1.7
# The function can be removed when we bump the requirements to >=1.7
def choice(a, size=None, replace=True, p=None, random_state=None):
    """
    choice(a, size=None, replace=True, p=None)

    Generates a random sample from a given 1-D array

    .. versionadded:: 1.7.0

    Parameters
    -----------
    a : 1-D array-like or int
        If an ndarray, a random sample is generated from its elements.
        If an int, the random sample is generated as if a was np.arange(n)

    size : int or tuple of ints, optional
        Output shape. Default is None, in which case a single value is
        returned.

    replace : boolean, optional
        Whether the sample is with or without replacement.

    p : 1-D array-like, optional
        The probabilities associated with each entry in a.
        If not given the sample assumes a uniform distribution over all
        entries in a.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    --------
    samples : 1-D ndarray, shape (size,)
    The generated random samples

    Raises
    -------
    ValueError
    If a is an int and less than zero, if a or p are not 1-dimensional,
    if a is an array-like of size 0, if p is not a vector of
    probabilities, if a and p have different lengths, or if
    replace=False and the sample size is greater than the population
    size

    See Also
    ---------
    randint, shuffle, permutation

    Examples
    ---------
    Generate a uniform random sample from np.arange(5) of size 3:

    >>> np.random.choice(5, 3)  # doctest: +SKIP
    array([0, 3, 4])
    >>> #This is equivalent to np.random.randint(0,5,3)

    Generate a non-uniform random sample from np.arange(5) of size 3:

    >>> np.random.choice(5, 3, p=[0.1, 0, 0.3, 0.6, 0])  # doctest: +SKIP
    array([3, 3, 0])

    Generate a uniform random sample from np.arange(5) of size 3 without
    replacement:

    >>> np.random.choice(5, 3, replace=False)  # doctest: +SKIP
    array([3,1,0])
    >>> #This is equivalent to np.random.shuffle(np.arange(5))[:3]

    Generate a non-uniform random sample from np.arange(5) of size
    3 without replacement:

    >>> np.random.choice(5, 3, replace=False, p=[0.1, 0, 0.3, 0.6, 0])
    ... # doctest: +SKIP
    array([2, 3, 0])

    Any of the above can be repeated with an arbitrary array-like
    instead of just integers. For instance:

    >>> aa_milne_arr = ['pooh', 'rabbit', 'piglet', 'Christopher']
    >>> np.random.choice(aa_milne_arr, 5, p=[0.5, 0.1, 0.1, 0.3])
    ... # doctest: +SKIP
    array(['pooh', 'pooh', 'pooh', 'Christopher', 'piglet'],
    dtype='|S11')

    """
    random_state = check_random_state(random_state)

    # Format and Verify input
    a = np.array(a, copy=False)
    if a.ndim == 0:
        try:
            # __index__ must return an integer by python rules.
            pop_size = operator.index(a.item())
        except TypeError:
            raise ValueError("a must be 1-dimensional or an integer")
        if pop_size <= 0:
            raise ValueError("a must be greater than 0")
    elif a.ndim != 1:
        raise ValueError("a must be 1-dimensional")
    else:
        pop_size = a.shape[0]
        if pop_size is 0:
            raise ValueError("a must be non-empty")

    if p is not None:
        p = np.array(p, dtype=np.double, ndmin=1, copy=False)
        if p.ndim != 1:
            raise ValueError("p must be 1-dimensional")
        if p.size != pop_size:
            raise ValueError("a and p must have same size")
        if np.any(p < 0):
            raise ValueError("probabilities are not non-negative")
        if not np.allclose(p.sum(), 1):
            raise ValueError("probabilities do not sum to 1")

    shape = size
    if shape is not None:
        size = np.prod(shape, dtype=np.intp)
    else:
        size = 1

    # Actual sampling
    if replace:
        if p is not None:
            cdf = p.cumsum()
            cdf /= cdf[-1]
            uniform_samples = random_state.random_sample(shape)
            idx = cdf.searchsorted(uniform_samples, side='right')
            # searchsorted returns a scalar
            idx = np.array(idx, copy=False)
        else:
            idx = random_state.randint(0, pop_size, size=shape)
    else:
        if size > pop_size:
            raise ValueError("Cannot take a larger sample than "
                             "population when 'replace=False'")

        if p is not None:
            if np.sum(p > 0) < size:
                raise ValueError("Fewer non-zero entries in p than size")
            n_uniq = 0
            p = p.copy()
            found = np.zeros(shape, dtype=np.int)
            flat_found = found.ravel()
            while n_uniq < size:
                x = random_state.rand(size - n_uniq)
                if n_uniq > 0:
                    p[flat_found[0:n_uniq]] = 0
                cdf = np.cumsum(p)
                cdf /= cdf[-1]
                new = cdf.searchsorted(x, side='right')
                _, unique_indices = np.unique(new, return_index=True)
                unique_indices.sort()
                new = new.take(unique_indices)
                flat_found[n_uniq:n_uniq + new.size] = new
                n_uniq += new.size
            idx = found
        else:
            idx = random_state.permutation(pop_size)[:size]
            if shape is not None:
                idx.shape = shape

    if shape is None and isinstance(idx, np.ndarray):
        # In most cases a scalar will have been made an array
        idx = idx.item(0)

    # Use samples as indices for a if a is array-like
    if a.ndim == 0:
        return idx

    if shape is not None and idx.ndim == 0:
        # If size == () then the user requested a 0-d array as opposed to
        # a scalar object when size is None. However a[idx] is always a
        # scalar and not an array. So this makes sure the result is an
        # array, taking into account that np.array(item) may not work
        # for object arrays.
        res = np.empty((), dtype=a.dtype)
        res[()] = a[idx]
        return res

    return a[idx]

def random_choice_csc(n_samples, classes, class_probability=None,
                      random_state=None):
    """Generate a sparse random matrix given column class distributions

    Parameters
    ----------
    n_samples : int,
        Number of samples to draw in each column.

    classes : list of size n_outputs of arrays of size (n_classes,)
        List of classes for each column.

    class_probability : list of size n_outputs of arrays of size (n_classes,)
        Optional (default=None). Class distribution of each column. If None the
        uniform distribution is assumed.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    random_matrix : sparse csc matrix of size (n_samples, n_outputs)

    """
    data = array.array('i')
    indices = array.array('i')
    indptr = array.array('i', [0])

    for j in range(len(classes)):
        classes[j] = np.asarray(classes[j])
        if classes[j].dtype.kind != 'i':
            raise ValueError("class dtype %s is not supported" %
                             classes[j].dtype)
        classes[j] = astype(classes[j], np.int64, copy=False)

        # use uniform distribution if no class_probability is given
        if class_probability is None:
            class_prob_j = np.empty(shape=classes[j].shape[0])
            class_prob_j.fill(1 / classes[j].shape[0])
        else:
            class_prob_j = np.asarray(class_probability[j])

        if np.sum(class_prob_j) != 1.0:
            raise ValueError("Probability array at index {0} does not sum to "
                             "one".format(j))

        if class_prob_j.shape[0] != classes[j].shape[0]:
            raise ValueError("classes[{0}] (length {1}) and "
                             "class_probability[{0}] (length {2}) have "
                             "different length.".format(j,
                                                        classes[j].shape[0],
                                                        class_prob_j.shape[0]))

        # If 0 is not present in the classes insert it with a probability 0.0
        if 0 not in classes[j]:
            classes[j] = np.insert(classes[j], 0, 0)
            class_prob_j = np.insert(class_prob_j, 0, 0.0)

        # If there are nonzero classes choose randomly using class_probability
        rng = check_random_state(random_state)
        if classes[j].shape[0] > 1:
            p_nonzero = 1 - class_prob_j[classes[j] == 0]
            nnz = int(n_samples * p_nonzero)
            ind_sample = sample_without_replacement(n_population=n_samples,
                                                    n_samples=nnz,
                                                    random_state=random_state)
            indices.extend(ind_sample)

            # Normalize probabilites for the nonzero elements
            classes_j_nonzero = classes[j] != 0
            class_probability_nz = class_prob_j[classes_j_nonzero]
            class_probability_nz_norm = (class_probability_nz /
                                         np.sum(class_probability_nz))
            classes_ind = np.searchsorted(class_probability_nz_norm.cumsum(),
                                          rng.rand(nnz))
            data.extend(classes[j][classes_j_nonzero][classes_ind])
        indptr.append(len(indices))

    return sp.csc_matrix((data, indices, indptr),
                         (n_samples, len(classes)),
                         dtype=int)


#get_ipython().magic(u'matplotlib inline')
import matplotlib.pyplot as plt
from scipy.spatial.distance import pdist, squareform
import scipy.io as sio
import h5py
import networkx as nx
import numpy as np
import gdist

def calcPaths(num):
    length = nx.all_pairs_dijkstra_path(G, num)
    length_paths = []
    for node in length:
        for target in length[node]:
            if len(length[node][target]) == num:
                length_paths.append(length[node][target])
    labeled_paths = labels[length_paths]
    same_labels = (squareform(pdist(labeled_paths)) < 1e-10).sum(axis=1)
    return length_paths, labeled_paths, same_labels

def uniqueRows(labeled_paths, same_labels, cutoff):
    a = labeled_paths[same_labels == cutoff]
    uRows = np.unique(a.view(np.dtype((np.void, a.dtype.itemsize*a.shape[1])))).view(a.dtype).reshape(-1, a.shape[1])
    return uRows
    print uRows

def removePaths(labeled_paths, same_labels, vals, row):
    ind = np.in1d(labeled_paths[:,row], vals).reshape(labeled_paths[:,row].shape)
    labeled_paths_new = labeled_paths[ind]
    same_labels_new = same_labels[ind]
    return ind, labeled_paths_new, same_labels_new

def removePathsInverse(labeled_paths, same_labels, vals, row):
    ind = np.in1d(labeled_paths[:,row], vals, invert=True).reshape(labeled_paths[:,row].shape)
    labeled_paths_new = labeled_paths[ind]
    same_labels_new = same_labels[ind]
    return ind, labeled_paths_new, same_labels_new

def printAll(labeled_paths,same_labels):
    print (('1 times:\n %s \n' % uniqueRows(labeled_paths,same_labels,1)))
    print (('2 times:\n %s \n' % uniqueRows(labeled_paths,same_labels,2)))
    print (('3 times:\n %s \n' % uniqueRows(labeled_paths,same_labels,3)))
    print (('4 times:\n %s \n' % uniqueRows(labeled_paths,same_labels,4)))
    print (('5 times:\n %s \n' % uniqueRows(labeled_paths,same_labels,5)))
    print (('6 times:\n %s \n' % uniqueRows(labeled_paths,same_labels,6)))

def labelHist(num,labeled_paths):
    a = np.zeros([num,18])
    for i in xrange(0,num):
        a[i] = np.histogram(labeled_paths[:,i],18, range=(1,18))[0]
    return a.transpose()

def calcPlace(b):
    c = np.zeros([len(b),1])
    for i in xrange(0,len(b)):
        if np.sum(b[i]) == 0:
            c[i] = 0
        else:
            c[i] = np.average(np.array(xrange(0,num)) + 1, weights=b[i])
    return c

'''load data:'''
fp = h5py.File('data/clus.mat')
fp.keys()
adj = fp['clus']['edge'][:]
labels = fp['clus']['edgeNet'][:].flatten()
G = nx.from_numpy_matrix(adj)

num=6
length_paths, labeled_paths, same_labels = calcPaths(num)
ind1, labeled_paths1, same_labels1 = removePaths(labeled_paths, same_labels, [7,12,14,15,16], 0)
ind2, labeled_paths2, same_labels2 = removePathsInverse(labeled_paths1, same_labels1, [7,12,14,15,16], 1)
ind3, labeled_paths3, same_labels3 = removePathsInverse(labeled_paths2, same_labels2, [7,12,14,15,16], 2)
ind4, labeled_paths4, same_labels4 = removePaths(labeled_paths3, same_labels3, [17,3,8], num-2)
ind5, labeled_paths5, same_labels5 = removePaths(labeled_paths4, same_labels4, [13], num-1)
print '\nNum = %s' % num
printAll(labeled_paths1, same_labels1)

'''histograms'''
indX, labeled_pathsX, same_labelsX = removePathsInverse(labeled_paths3, same_labels3, [7,12,14,15,16], 3)

num=5
length_paths, labeled_paths, same_labels = calcPaths(num)
ind1, labeled_paths1, same_labels1 =     removePaths(labeled_paths, same_labels, [7,12,14,15,16], 0)
a = labelHist(num,labeled_paths1)
print a
order = np.argsort(calcPlace(a), axis=0) + 1
fl = np.floor(np.sort(calcPlace(a), axis=0))
print '\nNum = %s' % num
print np.hstack((order, fl))

num=6
length_paths, labeled_paths, same_labels = calcPaths(num)
ind1, labeled_paths1, same_labels1 =     removePaths(labeled_paths, same_labels, [7,12,14,15,16], 0)
a = labelHist(num,labeled_paths1)
order = np.argsort(calcPlace(a), axis=0) + 1
fl = np.floor(np.sort(calcPlace(a), axis=0))
print '\nNum = %s' % num
print np.hstack((order, fl))

np.array(fl[np.argsort(order, axis=0)]).transpose()

np.array(length_paths)[same_labels == same_labels.max()]