#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from __future__ import division

from kudu.compat import unittest
from kudu.tests.util import TestScanBase
from kudu.tests.common import KuduTestBase, TimeoutError
import kudu
import datetime
import time
import pytest

class TestScanner(TestScanBase):

    @classmethod
    def setUpClass(self):
        super(TestScanner, self).setUpClass()

    def setUp(self):
        pass

    def test_scan_rows_basic(self):
        # Let's scan with no predicates
        scanner = self.table.scanner().open()

        tuples = scanner.read_all_tuples()
        self.assertEqual(sorted(tuples), self.tuples)

    def test_scan_rows_simple_predicate(self):
        key = self.table['key']
        preds = [key > 19, key < 50]

        def _read_predicates(preds):
            scanner = self.table.scanner()
            scanner.add_predicates(preds)
            scanner.open()
            return scanner.read_all_tuples()

        tuples = _read_predicates(preds)
        self.assertEqual(sorted(tuples), self.tuples[20:50])

        # verify predicates reusable
        tuples = _read_predicates(preds)
        self.assertEqual(sorted(tuples), self.tuples[20:50])

    def test_scan_limit(self):
        # Set limits both below and above the max number of rows.
        limits = [self.nrows - 1, self.nrows, self.nrows + 1]
        for limit in limits:
            scanner = self.table.scanner()
            scanner.set_limit(limit)
            tuples = scanner.read_all_tuples()
            self.assertEqual(len(tuples), min(limit, self.nrows))

    def test_scan_rows_string_predicate_and_projection(self):
        scanner = self.table.scanner()
        scanner.set_projected_column_names(['key', 'string_val'])
        sv = self.table['string_val']
        scanner.add_predicates([sv >= 'hello_20',
                                sv <= 'hello_22'])
        scanner.set_fault_tolerant()
        scanner.open()

        tuples = scanner.read_all_tuples()

        self.assertEqual(sorted(tuples), [(20, 'hello_20'), (22, 'hello_22')])

    def test_scan_rows_in_list_predicate(self):
        """
        Test scanner with an InList predicate and
        a string comparison predicate
        """
        key_list = [2, 98]
        scanner = self.table.scanner()
        scanner.set_fault_tolerant()\
            .add_predicates([
                self.table[0].in_list(key_list),
                self.table['string_val'] >= 'hello_9'
            ])
        scanner.open()

        tuples = scanner.read_all_tuples()

        self.assertEqual(tuples, [self.tuples[98]])

    def test_scan_rows_is_not_null_predicate(self):
        """
        Test scanner with an IsNotNull predicate on string_val column
        """
        pred = self.table['string_val'].is_not_null()
        scanner = self.table.scanner()
        scanner.add_predicate(pred)
        scanner.open()

        tuples = scanner.read_all_tuples()

        rows = [i for i in range(100) if i % 2 == 0]

        self.assertEqual(sorted(tuples), [self.tuples[i] for i in rows])

    def test_scan_rows_is_null_predicate(self):
        """
        Test scanner with an IsNull predicate on string_val column
        """
        pred = self.table['string_val'].is_null()
        scanner = self.table.scanner()
        scanner.add_predicate(pred)
        scanner.open()

        tuples = scanner.read_all_tuples()

        rows = [i for i in range(100) if i % 2 != 0]

        self.assertEqual(sorted(tuples), [self.tuples[i] for i in rows])

    def test_index_projection_with_schema(self):
        scanner = self.table.scanner()
        scanner.set_projected_column_indexes([0, 1])

        scanner.set_fault_tolerant()
        scanner.open()

        tuples = scanner.read_all_tuples()

        # Build schema to check against
        builder = kudu.schema_builder()
        builder.add_column('key', kudu.int32, nullable=False)
        builder.add_column('int_val', kudu.int32)
        builder.set_primary_keys(['key'])
        expected_schema = builder.build()

        # Build new schema from projection schema
        builder = kudu.schema_builder()
        for col in scanner.get_projection_schema():
            builder.copy_column(col)
        builder.set_primary_keys(['key'])
        new_schema = builder.build()

        self.assertEqual(tuples, [t[0:2] for t in self.tuples])
        self.assertTrue(expected_schema.equals(new_schema))

    def test_scan_with_bounds(self):
        scanner = self.table.scanner()
        scanner.set_fault_tolerant()\
            .add_lower_bound({'key': 50})\
            .add_exclusive_upper_bound({'key': 55})
        scanner.open()

        tuples = scanner.read_all_tuples()

        self.assertEqual(sorted(tuples), self.tuples[50:55])

    def test_scan_invalid_predicates(self):
        scanner = self.table.scanner()
        sv = self.table['string_val']

        with self.assertRaises(TypeError):
            scanner.add_predicates([sv >= None])

        with self.assertRaises(TypeError):
            scanner.add_predicates([sv >= 1])

        with self.assertRaises(TypeError):
            scanner.add_predicates([sv.in_list(['testing',
                                                datetime.datetime.utcnow()])])

        with self.assertRaises(TypeError):
            scanner.add_predicates([sv.in_list([
                'hello_20',
                120
            ])])

    def test_scan_batch_by_batch(self):
        scanner = self.table.scanner()
        scanner.set_fault_tolerant()
        lower_bound = scanner.new_bound()
        lower_bound['key'] = 10
        scanner.add_lower_bound(lower_bound)
        upper_bound = scanner.new_bound()
        upper_bound['key'] = 90
        scanner.add_exclusive_upper_bound(upper_bound)
        scanner.open()

        tuples = []
        while scanner.has_more_rows():
            batch = scanner.next_batch()
            tuples.extend(batch.as_tuples())

        self.assertEqual(sorted(tuples), self.tuples[10:90])

    def test_unixtime_micros(self):
        """
        Test setting and getting unixtime_micros fields
        """
        # Insert new rows
        self.insert_new_unixtime_micros_rows()

        # Validate results
        scanner = self.table.scanner()
        scanner.set_fault_tolerant().open()
        self.assertEqual(sorted(self.tuples), scanner.read_all_tuples())

    def test_read_mode(self):
        """
        Test scanning in latest, snapshot and read_your_writes read modes.
        """
        # Delete row
        self.delete_insert_row_for_read_test()

        # Check scanner results prior to delete
        scanner = self.table.scanner()
        scanner.set_read_mode('snapshot')\
            .set_snapshot(self.snapshot_timestamp)\
            .open()

        self.assertEqual(sorted(self.tuples[1:]), sorted(scanner.read_all_tuples()))

        # Check scanner results after delete with latest mode
        timeout = time.time() + 10
        check_tuples = []
        while check_tuples != sorted(self.tuples):
            if time.time() > timeout:
                raise TimeoutError("Could not validate results in allocated" +
                                   "time.")

            scanner = self.table.scanner()
            scanner.set_read_mode(kudu.READ_LATEST)\
                .open()
            check_tuples = sorted(scanner.read_all_tuples())
            # Avoid tight looping
            time.sleep(0.05)

        # Check scanner results after delete with read_your_writes mode
        scanner = self.table.scanner()
        scanner.set_read_mode('read_your_writes')\
            .open()

        self.assertEqual(sorted(self.tuples), sorted(scanner.read_all_tuples()))

    def test_resource_metrics_and_cache_blocks(self):
        """
        Test getting the resource metrics after scanning and
        setting the scanner to not cache blocks.
        """

        # Build scanner and read through all batches and retrieve metrics.
        scanner = self.table.scanner()
        scanner.set_fault_tolerant().set_cache_blocks(False).open()
        scanner.read_all_tuples()
        metrics = scanner.get_resource_metrics()

        # Confirm that the scanner returned cache hit and miss values.
        self.assertTrue('cfile_cache_hit_bytes' in metrics)
        self.assertTrue('cfile_cache_miss_bytes' in metrics)

    def verify_pred_type_scans(self, preds, row_indexes, count_only=False):
        # Using the incoming list of predicates, verify that the row returned
        # matches the inserted tuple at the row indexes specified in a
        # slice object
        scanner = self.type_table.scanner()
        scanner.set_fault_tolerant()
        scanner.add_predicates(preds)
        scanner.set_projected_column_names(self.projected_names_w_o_float)
        tuples = scanner.open().read_all_tuples()

        # verify rows
        if count_only:
            self.assertEqual(len(self.type_test_rows[row_indexes]), len(tuples))
        else:
            self.assertEqual(sorted(self.type_test_rows[row_indexes]), tuples)

    def test_unixtime_micros_pred(self):
        # Test unixtime_micros value predicate
        self._test_unixtime_micros_pred()

    def test_bool_pred(self):
        # Test a boolean value predicate
        self._test_bool_pred()

    def test_double_pred(self):
        # Test a double precision float predicate
        self._test_double_pred()

    def test_float_pred(self):
        # Test a single precision float predicate
        # Does a row check count only
        self._test_float_pred()

    def test_decimal_pred(self):
        if kudu.CLIENT_SUPPORTS_DECIMAL:
            # Test a decimal predicate
            self._test_decimal_pred()

    def test_binary_pred(self):
        # Test a binary predicate
        self._test_binary_pred()

    def test_scan_selection(self):
        """
        This test confirms that setting the scan selection policy on the
        scanner does not cause any errors. There is no way to confirm
        that the policy was actually set. This functionality is
        tested in the C++ test:
            ClientTest.TestReplicatedMultiTabletTableFailover.
        """

        for policy in ['leader', kudu.CLOSEST_REPLICA, 2]:
            scanner = self.table.scanner()
            scanner.set_selection(policy)
            scanner.open()
            self.assertEqual(sorted(scanner.read_all_tuples()),
                             sorted(self.tuples))

    @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS),
                        reason="Pandas required to run this test.")
    def test_scanner_to_pandas_types(self):
        """
        This test confirms that data types are converted as expected to Pandas.
        """
        import numpy as np
        scanner = self.type_table.scanner()
        df = scanner.to_pandas()
        types = df.dtypes

        if kudu.CLIENT_SUPPORTS_DECIMAL:
            self.assertEqual(types[0], np.int64)
            self.assertEqual(types[1], 'datetime64[ns, UTC]')
            self.assertEqual(types[2], np.object)
            self.assertEqual(types[3], np.object)
            self.assertEqual(types[4], np.bool)
            self.assertEqual(types[5], np.float64)
            self.assertEqual(types[6], np.int8)
            self.assertEqual(types[7], np.object)
            self.assertEqual(types[8], np.float32)
        else:
            self.assertEqual(types[0], np.int64)
            self.assertEqual(types[1], 'datetime64[ns, UTC]')
            self.assertEqual(types[2], np.object)
            self.assertEqual(types[3], np.bool)
            self.assertEqual(types[4], np.float64)
            self.assertEqual(types[5], np.int8)
            self.assertEqual(types[6], np.object)
            self.assertEqual(types[7], np.float32)

    @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS),
                        reason="Pandas required to run this test.")
    def test_scanner_to_pandas_row_count(self):
        """
        This test confirms that the record counts match between Pandas and the scanner.
        """
        scanner = self.type_table.scanner()
        scanner_count = len(scanner.read_all_tuples())
        scanner = self.type_table.scanner()
        df = scanner.to_pandas()
        self.assertEqual(scanner_count, df.shape[0])

    @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS),
                        reason="Pandas required to run this test.")
    def test_scanner_to_pandas_index(self):
        """
        This test confirms that an index is correctly applied.
        """
        scanner = self.type_table.scanner()
        df = scanner.to_pandas(index='key')
        self.assertEqual(df.index.name, 'key')
        self.assertEqual(list(df.index), [1, 2])

    @pytest.mark.skipif((not(kudu.CLIENT_SUPPORTS_PANDAS) or
                        (not(kudu.CLIENT_SUPPORTS_DECIMAL))),
                        reason="Pandas and Decimal support required to run this test.")
    def test_scanner_to_pandas_index(self):
        """
        This test confirms that a decimal column is coerced to a double when specified.
        """
        import numpy as np
        scanner = self.type_table.scanner()
        df = scanner.to_pandas(coerce_float=True)
        types = df.dtypes
        self.assertEqual(types[2], np.float64)


"""
==============================================
Face completion with a multi-output estimators
==============================================

This example shows the use of multi-output estimator to complete images.
The goal is to predict the lower half of a face given its upper half.

The first column of images shows true faces. The next columns illustrate
how extremely randomized trees, k nearest neighbors, linear
regression and ridge regression complete the lower half of those faces.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_olivetti_faces
from sklearn.utils.validation import check_random_state

from sklearn.ensemble import ExtraTreesRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import RidgeCV

# Load the faces datasets
data = fetch_olivetti_faces()
targets = data.target

data = data.images.reshape((len(data.images), -1))
train = data[targets < 30]
test = data[targets >= 30]  # Test on independent people

# Test on a subset of people
n_faces = 5
rng = check_random_state(4)
face_ids = rng.randint(test.shape[0], size=(n_faces, ))
test = test[face_ids, :]

n_pixels = data.shape[1]
X_train = train[:, :np.ceil(0.5 * n_pixels)]  # Upper half of the faces
y_train = train[:, np.floor(0.5 * n_pixels):]  # Lower half of the faces
X_test = test[:, :np.ceil(0.5 * n_pixels)]
y_test = test[:, np.floor(0.5 * n_pixels):]

# Fit estimators
ESTIMATORS = {
    "Extra trees": ExtraTreesRegressor(n_estimators=10, max_features=32,
                                       random_state=0),
    "K-nn": KNeighborsRegressor(),
    "Linear regression": LinearRegression(),
    "Ridge": RidgeCV(),
}

y_test_predict = dict()
for name, estimator in ESTIMATORS.items():
    estimator.fit(X_train, y_train)
    y_test_predict[name] = estimator.predict(X_test)

# Plot the completed faces
image_shape = (64, 64)

n_cols = 1 + len(ESTIMATORS)
plt.figure(figsize=(2. * n_cols, 2.26 * n_faces))
plt.suptitle("Face completion with multi-output estimators", size=16)

for i in range(n_faces):
    true_face = np.hstack((X_test[i], y_test[i]))

    if i:
        sub = plt.subplot(n_faces, n_cols, i * n_cols + 1)
    else:
        sub = plt.subplot(n_faces, n_cols, i * n_cols + 1,
                          title="true faces")

    sub.axis("off")
    sub.imshow(true_face.reshape(image_shape),
               cmap=plt.cm.gray,
               interpolation="nearest")

    for j, est in enumerate(sorted(ESTIMATORS)):
        completed_face = np.hstack((X_test[i], y_test_predict[est][i]))

        if i:
            sub = plt.subplot(n_faces, n_cols, i * n_cols + 2 + j)

        else:
            sub = plt.subplot(n_faces, n_cols, i * n_cols + 2 + j,
                              title=est)

        sub.axis("off")
        sub.imshow(completed_face.reshape(image_shape),
                   cmap=plt.cm.gray,
                   interpolation="nearest")

plt.show()


SentimentAnalysis - Polarity - Domain Specific Lexicon.py

import csv
import pandas as pd
import nltk
from nltk import FreqDist,ngrams
from nltk.corpus import stopwords
import string
from os import listdir
from os.path import isfile, join

def ngram_list(file,n):
    f = open(file,'rU')
    raw = f.read()
    raw = raw.replace('\n',' ')
    #raw = raw.decode('utf8')
    #raw = raw.decode("utf-8", 'ignore')
    ngramz = ngrams(raw.split(),n)
    return ngramz
def IsNotNull(value):
	    return value is not None and len(value) > 0

mypath = '/Users/francis/Documents/FORDHAM/2nd Term/Text Analytics/' #path where files are located
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]

dict_p = []
f = open('positive.txt', 'r')
for line in f:
	t = line.strip().lower()
	if IsNotNull(t):
		dict_p.append(t)
f.close

dict_n = []
f = open('negative.txt', 'r')
for line in f:
	t = line.strip().lower()
	if IsNotNull(t):
		dict_n.append(t)
f.close
totallist = []
rowlist = []
qa = 0
qb = 0
counti = 0
for i in onlyfiles:
    if i.endswith('.txt'):
        # get code
        j = i.replace('.txt','')
        # string filename
        file = mypath + str(i)
        print i

        f = open(file,'rU')
        raw = f.read()
        #print type(raw)
        raw = [w.translate(None, string.punctuation) for w in raw]
        raw = ''.join(raw)
    	raw = raw.replace('\n','')
    	raw = raw.replace(' ','')
    	#print raw
    	qa = 0
    	qb = 0
        for word in dict_p:
			if word in raw:
				qa += 1

    	for word in dict_n:
			if word in raw:
				qb += 1

    	qc = qa - qb

    	if qc > 0:
			sentiment = 'POSITIVE'
    	elif qc == 0:
			sentiment = 'NEUTRAL'
    	else:
			sentiment = 'NEGATIVE'

    	rowlist.append(i)
    	rowlist.append(qa)
    	rowlist.append(qb)
    	rowlist.append(qc)
    	rowlist.append(sentiment)

    	print counti
    	counti += 1
    	totallist.append(rowlist)

    	rowlist = []

    else:
        pass

labels = ('file', 'P', 'N', 'NET', 'SENTIMENT')
df = pd.DataFrame.from_records(totallist, columns = labels)

df.to_csv('oursentiment.csv', index = False)

#print dict_p

# allbigrams.append(ngram_list(file,2))
        # print i + ' BIGRAM - OK'
        # alltrigrams.append(ngram_list(file,3))
        # print i + ' TRIGRAM - OK'
        # allfourgrams.append(ngram_list(file,4))
        # print i + ' FOURGRAM - OK'
        # allfivegrams.append(ngram_list(file,5))
        # print i + ' TRIGRAM - OK'
        # allsixgrams.append(ngram_list(file,6))
        # print i + ' SIXGRAM - OK'
        # allsevengrams.append(ngram_list(file,7))
        # print i + ' SEVENGRAM - OK'
        # alleightgrams.append(ngram_list(file,8))
        # print i + ' EIGHTGRAM - OK'


"""Common tests for metaestimators"""

import functools

import numpy as np

from sklearn.base import BaseEstimator
from sklearn.externals.six import iterkeys
from sklearn.datasets import make_classification
from sklearn.utils.testing import assert_true, assert_false, assert_raises
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV, RandomizedSearchCV
from sklearn.feature_selection import RFE, RFECV
from sklearn.ensemble import BaggingClassifier

class DelegatorData(object):
    def __init__(self, name, construct, skip_methods=(),
                 fit_args=make_classification()):
        self.name = name
        self.construct = construct
        self.fit_args = fit_args
        self.skip_methods = skip_methods

DELEGATING_METAESTIMATORS = [
    DelegatorData('Pipeline', lambda est: Pipeline([('est', est)])),
    DelegatorData('GridSearchCV',
                  lambda est: GridSearchCV(
                      est, param_grid={'param': [5]}, cv=2),
                  skip_methods=['score']),
    DelegatorData('RandomizedSearchCV',
                  lambda est: RandomizedSearchCV(
                      est, param_distributions={'param': [5]}, cv=2, n_iter=1),
                  skip_methods=['score']),
    DelegatorData('RFE', RFE,
                  skip_methods=['transform', 'inverse_transform', 'score']),
    DelegatorData('RFECV', RFECV,
                  skip_methods=['transform', 'inverse_transform', 'score']),
    DelegatorData('BaggingClassifier', BaggingClassifier,
                  skip_methods=['transform', 'inverse_transform', 'score',
                                'predict_proba', 'predict_log_proba', 'predict'])
]

def test_metaestimator_delegation():
    # Ensures specified metaestimators have methods iff subestimator does
    def hides(method):
        @property
        def wrapper(obj):
            if obj.hidden_method == method.__name__:
                raise AttributeError('%r is hidden' % obj.hidden_method)
            return functools.partial(method, obj)
        return wrapper

    class SubEstimator(BaseEstimator):
        def __init__(self, param=1, hidden_method=None):
            self.param = param
            self.hidden_method = hidden_method

        def fit(self, X, y=None, *args, **kwargs):
            self.coef_ = np.arange(X.shape[1])
            return True

        def _check_fit(self):
            if not hasattr(self, 'coef_'):
                raise RuntimeError('Estimator is not fit')

        @hides
        def inverse_transform(self, X, *args, **kwargs):
            self._check_fit()
            return X

        @hides
        def transform(self, X, *args, **kwargs):
            self._check_fit()
            return X

        @hides
        def predict(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def predict_proba(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def predict_log_proba(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def decision_function(self, X, *args, **kwargs):
            self._check_fit()
            return np.ones(X.shape[0])

        @hides
        def score(self, X, *args, **kwargs):
            self._check_fit()
            return 1.0

    methods = [k for k in iterkeys(SubEstimator.__dict__)
               if not k.startswith('_') and not k.startswith('fit')]
    methods.sort()

    for delegator_data in DELEGATING_METAESTIMATORS:
        delegate = SubEstimator()
        delegator = delegator_data.construct(delegate)
        for method in methods:
            if method in delegator_data.skip_methods:
                continue
            assert_true(hasattr(delegate, method))
            assert_true(hasattr(delegator, method),
                        msg="%s does not have method %r when its delegate does"
                            % (delegator_data.name, method))
            # delegation before fit raises an exception
            assert_raises(Exception, getattr(delegator, method),
                          delegator_data.fit_args[0])

        delegator.fit(*delegator_data.fit_args)
        for method in methods:
            if method in delegator_data.skip_methods:
                continue
            # smoke test delegation
            getattr(delegator, method)(delegator_data.fit_args[0])

        for method in methods:
            if method in delegator_data.skip_methods:
                continue
            delegate = SubEstimator(hidden_method=method)
            delegator = delegator_data.construct(delegate)
            assert_false(hasattr(delegate, method))
            assert_false(hasattr(delegator, method),
                         msg="%s has method %r when its delegate does not"
                             % (delegator_data.name, method))


import itertools
import pickle

import numpy as np
from numpy.testing import assert_array_almost_equal

import scipy
from scipy.spatial.distance import cdist
from sklearn.neighbors.dist_metrics import DistanceMetric
from nose import SkipTest

def dist_func(x1, x2, p):
    return np.sum((x1 - x2) ** p) ** (1. / p)

def cmp_version(version1, version2):
    version1 = tuple(map(int, version1.split('.')[:2]))
    version2 = tuple(map(int, version2.split('.')[:2]))

    if version1 < version2:
        return -1
    elif version1 > version2:
        return 1
    else:
        return 0

class TestMetrics:
    def __init__(self, n1=20, n2=25, d=4, zero_frac=0.5,
                 rseed=0, dtype=np.float64):
        np.random.seed(rseed)
        self.X1 = np.random.random((n1, d)).astype(dtype)
        self.X2 = np.random.random((n2, d)).astype(dtype)

        # make boolean arrays: ones and zeros
        self.X1_bool = self.X1.round(0)
        self.X2_bool = self.X2.round(0)

        V = np.random.random((d, d))
        VI = np.dot(V, V.T)

        self.metrics = {'euclidean': {},
                        'cityblock': {},
                        'minkowski': dict(p=(1, 1.5, 2, 3)),
                        'chebyshev': {},
                        'seuclidean': dict(V=(np.random.random(d),)),
                        'wminkowski': dict(p=(1, 1.5, 3),
                                           w=(np.random.random(d),)),
                        'mahalanobis': dict(VI=(VI,)),
                        'hamming': {},
                        'canberra': {},
                        'braycurtis': {}}

        self.bool_metrics = ['matching', 'jaccard', 'dice',
                             'kulsinski', 'rogerstanimoto', 'russellrao',
                             'sokalmichener', 'sokalsneath']

    def test_cdist(self):
        for metric, argdict in self.metrics.items():
            keys = argdict.keys()
            for vals in itertools.product(*argdict.values()):
                kwargs = dict(zip(keys, vals))
                D_true = cdist(self.X1, self.X2, metric, **kwargs)
                yield self.check_cdist, metric, kwargs, D_true

        for metric in self.bool_metrics:
            D_true = cdist(self.X1_bool, self.X2_bool, metric)
            yield self.check_cdist_bool, metric, D_true

    def check_cdist(self, metric, kwargs, D_true):
        if metric == 'canberra' and cmp_version(scipy.__version__, '0.9') <= 0:
            raise SkipTest("Canberra distance incorrect in scipy < 0.9")
        dm = DistanceMetric.get_metric(metric, **kwargs)
        D12 = dm.pairwise(self.X1, self.X2)
        assert_array_almost_equal(D12, D_true)

    def check_cdist_bool(self, metric, D_true):
        dm = DistanceMetric.get_metric(metric)
        D12 = dm.pairwise(self.X1_bool, self.X2_bool)
        assert_array_almost_equal(D12, D_true)

    def test_pdist(self):
        for metric, argdict in self.metrics.items():
            keys = argdict.keys()
            for vals in itertools.product(*argdict.values()):
                kwargs = dict(zip(keys, vals))
                D_true = cdist(self.X1, self.X1, metric, **kwargs)
                yield self.check_pdist, metric, kwargs, D_true

        for metric in self.bool_metrics:
            D_true = cdist(self.X1_bool, self.X1_bool, metric)
            yield self.check_pdist_bool, metric, D_true

    def check_pdist(self, metric, kwargs, D_true):
        if metric == 'canberra' and cmp_version(scipy.__version__, '0.9') <= 0:
            raise SkipTest("Canberra distance incorrect in scipy < 0.9")
        dm = DistanceMetric.get_metric(metric, **kwargs)
        D12 = dm.pairwise(self.X1)
        assert_array_almost_equal(D12, D_true)

    def check_pdist_bool(self, metric, D_true):
        dm = DistanceMetric.get_metric(metric)
        D12 = dm.pairwise(self.X1_bool)
        assert_array_almost_equal(D12, D_true)

def test_haversine_metric():
    def haversine_slow(x1, x2):
        return 2 * np.arcsin(np.sqrt(np.sin(0.5 * (x1[0] - x2[0])) ** 2
                                     + np.cos(x1[0]) * np.cos(x2[0]) *
                                     np.sin(0.5 * (x1[1] - x2[1])) ** 2))

    X = np.random.random((10, 2))

    haversine = DistanceMetric.get_metric("haversine")

    D1 = haversine.pairwise(X)
    D2 = np.zeros_like(D1)
    for i, x1 in enumerate(X):
        for j, x2 in enumerate(X):
            D2[i, j] = haversine_slow(x1, x2)

    assert_array_almost_equal(D1, D2)
    assert_array_almost_equal(haversine.dist_to_rdist(D1),
                              np.sin(0.5 * D2) ** 2)

def test_pyfunc_metric():
    X = np.random.random((10, 3))

    euclidean = DistanceMetric.get_metric("euclidean")
    pyfunc = DistanceMetric.get_metric("pyfunc", func=dist_func, p=2)

    # Check if both callable metric and predefined metric initialized
    # DistanceMetric object is picklable
    euclidean_pkl = pickle.loads(pickle.dumps(euclidean))
    pyfunc_pkl = pickle.loads(pickle.dumps(pyfunc))

    D1 = euclidean.pairwise(X)
    D2 = pyfunc.pairwise(X)

    D1_pkl = euclidean_pkl.pairwise(X)
    D2_pkl = pyfunc_pkl.pairwise(X)

    assert_array_almost_equal(D1, D2)
    assert_array_almost_equal(D1_pkl, D2_pkl)


import operator

import numpy as np
from pandas._libs import index as libindex

from pandas import compat
from pandas.compat.numpy import function as nv
from pandas.core.dtypes.generic import ABCCategorical, ABCSeries
from pandas.core.dtypes.dtypes import CategoricalDtype
from pandas.core.dtypes.common import (
    is_categorical_dtype,
    ensure_platform_int,
    is_list_like,
    is_interval_dtype,
    is_scalar)
from pandas.core.dtypes.missing import array_equivalent, isna
from pandas.core.algorithms import take_1d

from pandas.util._decorators import Appender, cache_readonly
from pandas.core.config import get_option
from pandas.core.indexes.base import Index, _index_shared_docs
from pandas.core import accessor
import pandas.core.common as com
import pandas.core.missing as missing
import pandas.core.indexes.base as ibase
from pandas.core.arrays.categorical import Categorical, contains

_index_doc_kwargs = dict(ibase._index_doc_kwargs)
_index_doc_kwargs.update(dict(target_klass='CategoricalIndex'))

class CategoricalIndex(Index, accessor.PandasDelegate):
    """

    Immutable Index implementing an ordered, sliceable set. CategoricalIndex
    represents a sparsely populated Index with an underlying Categorical.

    Parameters
    ----------
    data : array-like or Categorical, (1-dimensional)
    categories : optional, array-like
        categories for the CategoricalIndex
    ordered : boolean,
        designating if the categories are ordered
    copy : bool
        Make a copy of input ndarray
    name : object
        Name to be stored in the index

    Attributes
    ----------
    codes
    categories
    ordered

    Methods
    -------
    rename_categories
    reorder_categories
    add_categories
    remove_categories
    remove_unused_categories
    set_categories
    as_ordered
    as_unordered
    map

    See Also
    --------
    Categorical, Index
    """

    _typ = 'categoricalindex'
    _engine_type = libindex.Int64Engine
    _attributes = ['name']

    def __new__(cls, data=None, categories=None, ordered=None, dtype=None,
                copy=False, name=None, fastpath=False):

        if fastpath:
            return cls._simple_new(data, name=name, dtype=dtype)

        if name is None and hasattr(data, 'name'):
            name = data.name

        if isinstance(data, ABCCategorical):
            data = cls._create_categorical(data, categories, ordered,
                                           dtype)
        elif isinstance(data, CategoricalIndex):
            data = data._data
            data = cls._create_categorical(data, categories, ordered,
                                           dtype)
        else:

            # don't allow scalars
            # if data is None, then categories must be provided
            if is_scalar(data):
                if data is not None or categories is None:
                    cls._scalar_data_error(data)
                data = []
            data = cls._create_categorical(data, categories, ordered,
                                           dtype)

        if copy:
            data = data.copy()

        return cls._simple_new(data, name=name)

    def _create_from_codes(self, codes, categories=None, ordered=None,
                           name=None):
        """
        *this is an internal non-public method*

        create the correct categorical from codes

        Parameters
        ----------
        codes : new codes
        categories : optional categories, defaults to existing
        ordered : optional ordered attribute, defaults to existing
        name : optional name attribute, defaults to existing

        Returns
        -------
        CategoricalIndex
        """

        if categories is None:
            categories = self.categories
        if ordered is None:
            ordered = self.ordered
        if name is None:
            name = self.name
        cat = Categorical.from_codes(codes, categories=categories,
                                     ordered=self.ordered)
        return CategoricalIndex(cat, name=name)

    @classmethod
    def _create_categorical(cls, data, categories=None, ordered=None,
                            dtype=None):
        """
        *this is an internal non-public method*

        create the correct categorical from data and the properties

        Parameters
        ----------
        data : data for new Categorical
        categories : optional categories, defaults to existing
        ordered : optional ordered attribute, defaults to existing
        dtype : CategoricalDtype, defaults to existing

        Returns
        -------
        Categorical
        """
        if (isinstance(data, (cls, ABCSeries)) and
                is_categorical_dtype(data)):
            data = data.values

        if not isinstance(data, ABCCategorical):
            if ordered is None and dtype is None:
                ordered = False
            data = Categorical(data, categories=categories, ordered=ordered,
                               dtype=dtype)
        else:
            if categories is not None:
                data = data.set_categories(categories, ordered=ordered)
            elif ordered is not None and ordered != data.ordered:
                data = data.set_ordered(ordered)
            if isinstance(dtype, CategoricalDtype) and dtype != data.dtype:
                # we want to silently ignore dtype='category'
                data = data._set_dtype(dtype)
        return data

    @classmethod
    def _simple_new(cls, values, name=None, categories=None, ordered=None,
                    dtype=None, **kwargs):
        result = object.__new__(cls)

        values = cls._create_categorical(values, categories, ordered,
                                         dtype=dtype)
        result._data = values
        result.name = name
        for k, v in compat.iteritems(kwargs):
            setattr(result, k, v)

        result._reset_identity()
        return result

    @Appender(_index_shared_docs['_shallow_copy'])
    def _shallow_copy(self, values=None, categories=None, ordered=None,
                      dtype=None, **kwargs):
        # categories and ordered can't be part of attributes,
        # as these are properties
        # we want to reuse self.dtype if possible, i.e. neither are
        # overridden.
        if dtype is not None and (categories is not None or
                                  ordered is not None):
            raise TypeError("Cannot specify both `dtype` and `categories` "
                            "or `ordered`")

        if categories is None and ordered is None:
            dtype = self.dtype if dtype is None else dtype
            return super(CategoricalIndex, self)._shallow_copy(
                values=values, dtype=dtype, **kwargs)
        if categories is None:
            categories = self.categories
        if ordered is None:
            ordered = self.ordered

        return super(CategoricalIndex, self)._shallow_copy(
            values=values, categories=categories,
            ordered=ordered, **kwargs)

    def _is_dtype_compat(self, other):
        """
        *this is an internal non-public method*

        provide a comparison between the dtype of self and other (coercing if
        needed)

        Raises
        ------
        TypeError if the dtypes are not compatible
        """
        if is_categorical_dtype(other):
            if isinstance(other, CategoricalIndex):
                other = other._values
            if not other.is_dtype_equal(self):
                raise TypeError("categories must match existing categories "
                                "when appending")
        else:
            values = other
            if not is_list_like(values):
                values = [values]
            other = CategoricalIndex(self._create_categorical(
                other, dtype=self.dtype))
            if not other.isin(values).all():
                raise TypeError("cannot append a non-category item to a "
                                "CategoricalIndex")

        return other

    def equals(self, other):
        """
        Determines if two CategorialIndex objects contain the same elements.
        """
        if self.is_(other):
            return True

        if not isinstance(other, Index):
            return False

        try:
            other = self._is_dtype_compat(other)
            return array_equivalent(self._data, other)
        except (TypeError, ValueError):
            pass

        return False

    @property
    def _formatter_func(self):
        return self.categories._formatter_func

    def _format_attrs(self):
        """
        Return a list of tuples of the (attr,formatted_value)
        """
        max_categories = (10 if get_option("display.max_categories") == 0 else
                          get_option("display.max_categories"))
        attrs = [
            ('categories',
             ibase.default_pprint(self.categories,
                                  max_seq_items=max_categories)),
            ('ordered', self.ordered)]
        if self.name is not None:
            attrs.append(('name', ibase.default_pprint(self.name)))
        attrs.append(('dtype', "'%s'" % self.dtype.name))
        max_seq_items = get_option('display.max_seq_items') or len(self)
        if len(self) > max_seq_items:
            attrs.append(('length', len(self)))
        return attrs

    @property
    def inferred_type(self):
        return 'categorical'

    @property
    def values(self):
        """ return the underlying data, which is a Categorical """
        return self._data

    @property
    def itemsize(self):
        # Size of the items in categories, not codes.
        return self.values.itemsize

    def get_values(self):
        """ return the underlying data as an ndarray """
        return self._data.get_values()

    def tolist(self):
        return self._data.tolist()

    @property
    def codes(self):
        return self._data.codes

    @property
    def categories(self):
        return self._data.categories

    @property
    def ordered(self):
        return self._data.ordered

    def _reverse_indexer(self):
        return self._data._reverse_indexer()

    @Appender(_index_shared_docs['__contains__'] % _index_doc_kwargs)
    def __contains__(self, key):
        # if key is a NaN, check if any NaN is in self.
        if isna(key):
            return self.hasnans

        return contains(self, key, container=self._engine)

    @Appender(_index_shared_docs['contains'] % _index_doc_kwargs)
    def contains(self, key):
        return key in self

    def __array__(self, dtype=None):
        """ the array interface, return my values """
        return np.array(self._data, dtype=dtype)

    @Appender(_index_shared_docs['astype'])
    def astype(self, dtype, copy=True):
        if is_interval_dtype(dtype):
            from pandas import IntervalIndex
            return IntervalIndex(np.array(self))
        elif is_categorical_dtype(dtype):
            # GH 18630
            dtype = self.dtype.update_dtype(dtype)
            if dtype == self.dtype:
                return self.copy() if copy else self

        return super(CategoricalIndex, self).astype(dtype=dtype, copy=copy)

    @cache_readonly
    def _isnan(self):
        """ return if each value is nan"""
        return self._data.codes == -1

    @Appender(ibase._index_shared_docs['fillna'])
    def fillna(self, value, downcast=None):
        self._assert_can_do_op(value)
        return CategoricalIndex(self._data.fillna(value), name=self.name)

    def argsort(self, *args, **kwargs):
        return self.values.argsort(*args, **kwargs)

    @cache_readonly
    def _engine(self):

        # we are going to look things up with the codes themselves
        return self._engine_type(lambda: self.codes.astype('i8'), len(self))

    # introspection
    @cache_readonly
    def is_unique(self):
        return self._engine.is_unique

    @property
    def is_monotonic_increasing(self):
        return self._engine.is_monotonic_increasing

    @property
    def is_monotonic_decreasing(self):
        return self._engine.is_monotonic_decreasing

    @Appender(_index_shared_docs['index_unique'] % _index_doc_kwargs)
    def unique(self, level=None):
        if level is not None:
            self._validate_index_level(level)
        result = self.values.unique()
        # CategoricalIndex._shallow_copy keeps original categories
        # and ordered if not otherwise specified
        return self._shallow_copy(result, categories=result.categories,
                                  ordered=result.ordered)

    @Appender(Index.duplicated.__doc__)
    def duplicated(self, keep='first'):
        from pandas._libs.hashtable import duplicated_int64
        codes = self.codes.astype('i8')
        return duplicated_int64(codes, keep)

    def _to_safe_for_reshape(self):
        """ convert to object if we are a categorical """
        return self.astype('object')

    def get_loc(self, key, method=None):
        """
        Get integer location, slice or boolean mask for requested label.

        Parameters
        ----------
        key : label
        method : {None}
            * default: exact matches only.

        Returns
        -------
        loc : int if unique index, slice if monotonic index, else mask

        Examples
        ---------
        >>> unique_index = pd.CategoricalIndex(list('abc'))
        >>> unique_index.get_loc('b')
        1

        >>> monotonic_index = pd.CategoricalIndex(list('abbc'))
        >>> monotonic_index.get_loc('b')
        slice(1, 3, None)

        >>> non_monotonic_index = pd.CategoricalIndex(list('abcb'))
        >>> non_monotonic_index.get_loc('b')
        array([False,  True, False,  True], dtype=bool)
        """
        codes = self.categories.get_loc(key)
        if (codes == -1):
            raise KeyError(key)
        return self._engine.get_loc(codes)

    def get_value(self, series, key):
        """
        Fast lookup of value from 1-dimensional ndarray. Only use this if you
        know what you're doing
        """
        try:
            k = com._values_from_object(key)
            k = self._convert_scalar_indexer(k, kind='getitem')
            indexer = self.get_loc(k)
            return series.iloc[indexer]
        except (KeyError, TypeError):
            pass

        # we might be a positional inexer
        return super(CategoricalIndex, self).get_value(series, key)

    def _can_reindex(self, indexer):
        """ always allow reindexing """
        pass

    @Appender(_index_shared_docs['where'])
    def where(self, cond, other=None):
        if other is None:
            other = self._na_value
        values = np.where(cond, self.values, other)

        cat = Categorical(values,
                          categories=self.categories,
                          ordered=self.ordered)
        return self._shallow_copy(cat, **self._get_attributes_dict())

    def reindex(self, target, method=None, level=None, limit=None,
                tolerance=None):
        """
        Create index with target's values (move/add/delete values as necessary)

        Returns
        -------
        new_index : pd.Index
            Resulting index
        indexer : np.ndarray or None
            Indices of output values in original index

        """

        if method is not None:
            raise NotImplementedError("argument method is not implemented for "
                                      "CategoricalIndex.reindex")
        if level is not None:
            raise NotImplementedError("argument level is not implemented for "
                                      "CategoricalIndex.reindex")
        if limit is not None:
            raise NotImplementedError("argument limit is not implemented for "
                                      "CategoricalIndex.reindex")

        target = ibase.ensure_index(target)

        if not is_categorical_dtype(target) and not target.is_unique:
            raise ValueError("cannot reindex with a non-unique indexer")

        indexer, missing = self.get_indexer_non_unique(np.array(target))

        if len(self.codes):
            new_target = self.take(indexer)
        else:
            new_target = target

        # filling in missing if needed
        if len(missing):
            cats = self.categories.get_indexer(target)

            if (cats == -1).any():
                # coerce to a regular index here!
                result = Index(np.array(self), name=self.name)
                new_target, indexer, _ = result._reindex_non_unique(
                    np.array(target))
            else:

                codes = new_target.codes.copy()
                codes[indexer == -1] = cats[missing]
                new_target = self._create_from_codes(codes)

        # we always want to return an Index type here
        # to be consistent with .reindex for other index types (e.g. they don't
        # coerce based on the actual values, only on the dtype)
        # unless we had an initial Categorical to begin with
        # in which case we are going to conform to the passed Categorical
        new_target = np.asarray(new_target)
        if is_categorical_dtype(target):
            new_target = target._shallow_copy(new_target, name=self.name)
        else:
            new_target = Index(new_target, name=self.name)

        return new_target, indexer

    def _reindex_non_unique(self, target):
        """ reindex from a non-unique; which CategoricalIndex's are almost
        always
        """
        new_target, indexer = self.reindex(target)
        new_indexer = None

        check = indexer == -1
        if check.any():
            new_indexer = np.arange(len(self.take(indexer)))
            new_indexer[check] = -1

        cats = self.categories.get_indexer(target)
        if not (cats == -1).any():
            # .reindex returns normal Index. Revert to CategoricalIndex if
            # all targets are included in my categories
            new_target = self._shallow_copy(new_target)

        return new_target, indexer, new_indexer

    @Appender(_index_shared_docs['get_indexer'] % _index_doc_kwargs)
    def get_indexer(self, target, method=None, limit=None, tolerance=None):
        from pandas.core.arrays.categorical import _recode_for_categories

        method = missing.clean_reindex_fill_method(method)
        target = ibase.ensure_index(target)

        if self.is_unique and self.equals(target):
            return np.arange(len(self), dtype='intp')

        if method == 'pad' or method == 'backfill':
            raise NotImplementedError("method='pad' and method='backfill' not "
                                      "implemented yet for CategoricalIndex")
        elif method == 'nearest':
            raise NotImplementedError("method='nearest' not implemented yet "
                                      'for CategoricalIndex')

        if (isinstance(target, CategoricalIndex) and
                self.values.is_dtype_equal(target)):
            if self.values.equals(target.values):
                # we have the same codes
                codes = target.codes
            else:
                codes = _recode_for_categories(target.codes,
                                               target.categories,
                                               self.values.categories)
        else:
            if isinstance(target, CategoricalIndex):
                code_indexer = self.categories.get_indexer(target.categories)
                codes = take_1d(code_indexer, target.codes, fill_value=-1)
            else:
                codes = self.categories.get_indexer(target)

        indexer, _ = self._engine.get_indexer_non_unique(codes)
        return ensure_platform_int(indexer)

    @Appender(_index_shared_docs['get_indexer_non_unique'] % _index_doc_kwargs)
    def get_indexer_non_unique(self, target):
        target = ibase.ensure_index(target)

        if isinstance(target, CategoricalIndex):
            # Indexing on codes is more efficient if categories are the same:
            if target.categories is self.categories:
                target = target.codes
                indexer, missing = self._engine.get_indexer_non_unique(target)
                return ensure_platform_int(indexer), missing
            target = target.values

        codes = self.categories.get_indexer(target)
        indexer, missing = self._engine.get_indexer_non_unique(codes)
        return ensure_platform_int(indexer), missing

    @Appender(_index_shared_docs['_convert_scalar_indexer'])
    def _convert_scalar_indexer(self, key, kind=None):
        if self.categories._defer_to_indexing:
            return self.categories._convert_scalar_indexer(key, kind=kind)

        return super(CategoricalIndex, self)._convert_scalar_indexer(
            key, kind=kind)

    @Appender(_index_shared_docs['_convert_list_indexer'])
    def _convert_list_indexer(self, keyarr, kind=None):
        # Return our indexer or raise if all of the values are not included in
        # the categories

        if self.categories._defer_to_indexing:
            indexer = self.categories._convert_list_indexer(keyarr, kind=kind)
            return Index(self.codes).get_indexer_for(indexer)

        indexer = self.categories.get_indexer(np.asarray(keyarr))
        if (indexer == -1).any():
            raise KeyError(
                "a list-indexer must only "
                "include values that are "
                "in the categories")

        return self.get_indexer(keyarr)

    @Appender(_index_shared_docs['_convert_arr_indexer'])
    def _convert_arr_indexer(self, keyarr):
        keyarr = com._asarray_tuplesafe(keyarr)

        if self.categories._defer_to_indexing:
            return keyarr

        return self._shallow_copy(keyarr)

    @Appender(_index_shared_docs['_convert_index_indexer'])
    def _convert_index_indexer(self, keyarr):
        return self._shallow_copy(keyarr)

    @Appender(_index_shared_docs['take'] % _index_doc_kwargs)
    def take(self, indices, axis=0, allow_fill=True,
             fill_value=None, **kwargs):
        nv.validate_take(tuple(), kwargs)
        indices = ensure_platform_int(indices)
        taken = self._assert_take_fillable(self.codes, indices,
                                           allow_fill=allow_fill,
                                           fill_value=fill_value,
                                           na_value=-1)
        return self._create_from_codes(taken)

    def is_dtype_equal(self, other):
        return self._data.is_dtype_equal(other)

    take_nd = take

    def map(self, mapper):
        """
        Map values using input correspondence (a dict, Series, or function).

        Maps the values (their categories, not the codes) of the index to new
        categories. If the mapping correspondence is one-to-one the result is a
        :class:`~pandas.CategoricalIndex` which has the same order property as
        the original, otherwise an :class:`~pandas.Index` is returned.

        If a `dict` or :class:`~pandas.Series` is used any unmapped category is
        mapped to `NaN`. Note that if this happens an :class:`~pandas.Index`
        will be returned.

        Parameters
        ----------
        mapper : function, dict, or Series
            Mapping correspondence.

        Returns
        -------
        pandas.CategoricalIndex or pandas.Index
            Mapped index.

        See Also
        --------
        Index.map : Apply a mapping correspondence on an
            :class:`~pandas.Index`.
        Series.map : Apply a mapping correspondence on a
            :class:`~pandas.Series`.
        Series.apply : Apply more complex functions on a
            :class:`~pandas.Series`.

        Examples
        --------
        >>> idx = pd.CategoricalIndex(['a', 'b', 'c'])
        >>> idx
        CategoricalIndex(['a', 'b', 'c'], categories=['a', 'b', 'c'],
                         ordered=False, dtype='category')
        >>> idx.map(lambda x: x.upper())
        CategoricalIndex(['A', 'B', 'C'], categories=['A', 'B', 'C'],
                         ordered=False, dtype='category')
        >>> idx.map({'a': 'first', 'b': 'second', 'c': 'third'})
        CategoricalIndex(['first', 'second', 'third'], categories=['first',
                         'second', 'third'], ordered=False, dtype='category')

        If the mapping is one-to-one the ordering of the categories is
        preserved:

        >>> idx = pd.CategoricalIndex(['a', 'b', 'c'], ordered=True)
        >>> idx
        CategoricalIndex(['a', 'b', 'c'], categories=['a', 'b', 'c'],
                         ordered=True, dtype='category')
        >>> idx.map({'a': 3, 'b': 2, 'c': 1})
        CategoricalIndex([3, 2, 1], categories=[3, 2, 1], ordered=True,
                         dtype='category')

        If the mapping is not one-to-one an :class:`~pandas.Index` is returned:

        >>> idx.map({'a': 'first', 'b': 'second', 'c': 'first'})
        Index(['first', 'second', 'first'], dtype='object')

        If a `dict` is used, all unmapped categories are mapped to `NaN` and
        the result is an :class:`~pandas.Index`:

        >>> idx.map({'a': 'first', 'b': 'second'})
        Index(['first', 'second', nan], dtype='object')
        """
        return self._shallow_copy_with_infer(self.values.map(mapper))

    def delete(self, loc):
        """
        Make new Index with passed location(-s) deleted

        Returns
        -------
        new_index : Index
        """
        return self._create_from_codes(np.delete(self.codes, loc))

    def insert(self, loc, item):
        """
        Make new Index inserting new item at location. Follows
        Python list.append semantics for negative values

        Parameters
        ----------
        loc : int
        item : object

        Returns
        -------
        new_index : Index

        Raises
        ------
        ValueError if the item is not in the categories

        """
        code = self.categories.get_indexer([item])
        if (code == -1) and not (is_scalar(item) and isna(item)):
            raise TypeError("cannot insert an item into a CategoricalIndex "
                            "that is not already an existing category")

        codes = self.codes
        codes = np.concatenate((codes[:loc], code, codes[loc:]))
        return self._create_from_codes(codes)

    def _concat(self, to_concat, name):
        # if calling index is category, don't check dtype of others
        return CategoricalIndex._concat_same_dtype(self, to_concat, name)

    def _concat_same_dtype(self, to_concat, name):
        """
        Concatenate to_concat which has the same class
        ValueError if other is not in the categories
        """
        to_concat = [self._is_dtype_compat(c) for c in to_concat]
        codes = np.concatenate([c.codes for c in to_concat])
        result = self._create_from_codes(codes, name=name)
        # if name is None, _create_from_codes sets self.name
        result.name = name
        return result

    def _codes_for_groupby(self, sort, observed):
        """ Return a Categorical adjusted for groupby """
        return self.values._codes_for_groupby(sort, observed)

    @classmethod
    def _add_comparison_methods(cls):
        """ add in comparison methods """

        def _make_compare(op):
            opname = '__{op}__'.format(op=op.__name__)

            def _evaluate_compare(self, other):

                # if we have a Categorical type, then must have the same
                # categories
                if isinstance(other, CategoricalIndex):
                    other = other._values
                elif isinstance(other, Index):
                    other = self._create_categorical(
                        other._values, dtype=self.dtype)

                if isinstance(other, (ABCCategorical, np.ndarray,
                                      ABCSeries)):
                    if len(self.values) != len(other):
                        raise ValueError("Lengths must match to compare")

                if isinstance(other, ABCCategorical):
                    if not self.values.is_dtype_equal(other):
                        raise TypeError("categorical index comparisons must "
                                        "have the same categories and ordered "
                                        "attributes")

                result = op(self.values, other)
                if isinstance(result, ABCSeries):
                    # Dispatch to pd.Categorical returned NotImplemented
                    # and we got a Series back; down-cast to ndarray
                    result = result.values
                return result

            return compat.set_function_name(_evaluate_compare, opname, cls)

        cls.__eq__ = _make_compare(operator.eq)
        cls.__ne__ = _make_compare(operator.ne)
        cls.__lt__ = _make_compare(operator.lt)
        cls.__gt__ = _make_compare(operator.gt)
        cls.__le__ = _make_compare(operator.le)
        cls.__ge__ = _make_compare(operator.ge)

    def _delegate_method(self, name, *args, **kwargs):
        """ method delegation to the ._values """
        method = getattr(self._values, name)
        if 'inplace' in kwargs:
            raise ValueError("cannot use inplace with CategoricalIndex")
        res = method(*args, **kwargs)
        if is_scalar(res):
            return res
        return CategoricalIndex(res, name=self.name)

    @classmethod
    def _add_accessors(cls):
        """ add in Categorical accessor methods """

        CategoricalIndex._add_delegate_accessors(
            delegate=Categorical, accessors=["rename_categories",
                                             "reorder_categories",
                                             "add_categories",
                                             "remove_categories",
                                             "remove_unused_categories",
                                             "set_categories",
                                             "as_ordered", "as_unordered",
                                             "min", "max"],
            typ='method', overwrite=True)

CategoricalIndex._add_numeric_methods_add_sub_disabled()
CategoricalIndex._add_numeric_methods_disabled()
CategoricalIndex._add_logical_methods_disabled()
CategoricalIndex._add_comparison_methods()
CategoricalIndex._add_accessors()


#! /usr/bin/env python

#  Copyright (C) 2010 Antoine Drouin
#
# This file is part of Paparazzi.
#
# Paparazzi is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# Paparazzi is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Paparazzi; see the file COPYING.  If not, write to
# the Free Software Foundation, 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
#

#
# calibrate gyrometers using turntable measurements
#

from __future__ import print_function, division

from optparse import OptionParser
import os
import sys

from scipy import linspace, polyval, stats

import matplotlib.pyplot as plt

import calibration_utils

#
# lisa 3
# p : a=-4511.16 b=31948.34, std error= 0.603
# q : a=-4598.46 b=31834.48, std error= 0.734
# r : a=-4525.63 b=32687.95, std error= 0.624
#
# lisa 4
# p : a=-4492.05 b=32684.94, std error= 0.600
# q : a=-4369.63 b=33260.96, std error= 0.710
# r : a=-4577.13 b=32707.72, std error= 0.730
#
# crista
# p : a= 3864.82 b=31288.09, std error= 0.866
# q : a= 3793.71 b=32593.89, std error= 3.070
# r : a= 3817.11 b=32709.70, std error= 3.296
#

def main():
    usage = "usage: %prog --id <ac_id> --tt_id <tt_id> --axis <axis> [options] log_filename.data" + "\n" + "Run %prog --help to list the options."
    parser = OptionParser(usage)
    parser.add_option("-i", "--id", dest="ac_id",
                      action="store", type=int, default=-1,
                      help="aircraft id to use")
    parser.add_option("-t", "--tt_id", dest="tt_id",
                      action="store", type=int, default=-1,
                      help="turntable id to use")
    parser.add_option("-a", "--axis", dest="axis",
                      type="choice", choices=['p', 'q', 'r'],
                      help="axis to calibrate (p, q, r)",
                      action="store")
    parser.add_option("-v", "--verbose",
                      action="store_true", dest="verbose")
    (options, args) = parser.parse_args()
    if len(args) != 1:
        parser.error("incorrect number of arguments")
    else:
        if os.path.isfile(args[0]):
            filename = args[0]
        else:
            print(args[0] + " not found")
            sys.exit(1)
    if not filename.endswith(".data"):
        parser.error("Please specify a *.data log file")

    if options.ac_id < 0 or options.ac_id > 255:
        parser.error("Specify a valid aircraft id number!")
    if options.tt_id < 0 or options.tt_id > 255:
        parser.error("Specify a valid turntable id number!")
    if options.verbose:
        print("reading file "+filename+" for aircraft "+str(options.ac_id)+" and turntable "+str(options.tt_id))

    samples = calibration_utils.read_turntable_log(options.ac_id, options.tt_id, filename, 1, 7)

    if len(samples) == 0:
        print("Error: found zero matching messages in log file!")
        print("Was looking for IMU_TURNTABLE from id: "+str(options.tt_id)+" and IMU_GYRO_RAW from id: "+str(options.ac_id)+" in file "+filename)
        sys.exit(1)
    if options.verbose:
        print("found "+str(len(samples))+" records")

    if options.axis == 'p':
        axis_idx = 1
    elif options.axis == 'q':
        axis_idx = 2
    elif options.axis == 'r':
        axis_idx = 3
    else:
        parser.error("Specify a valid axis!")

    #Linear regression using stats.linregress
    t = samples[:, 0]
    xn = samples[:, axis_idx]
    (a_s, b_s, r, tt, stderr) = stats.linregress(t, xn)
    print('Linear regression using stats.linregress')
    print(('regression: a=%.2f b=%.2f, std error= %.3f' % (a_s, b_s, stderr)))
    print(('<define name="GYRO_X_NEUTRAL" value="%d"/>' % (b_s)))
    print(('<define name="GYRO_X_SENS" value="%f" integer="16"/>' % (pow(2, 12)/a_s)))

    #
    # overlay fited value
    #
    ovl_omega = linspace(1, 7.5, 10)
    ovl_adc = polyval([a_s, b_s], ovl_omega)

    plt.title('Linear Regression Example')
    plt.subplot(3, 1, 1)
    plt.plot(samples[:, 1])
    plt.plot(samples[:, 2])
    plt.plot(samples[:, 3])
    plt.legend(['p', 'q', 'r'])

    plt.subplot(3, 1, 2)
    plt.plot(samples[:, 0])

    plt.subplot(3, 1, 3)
    plt.plot(samples[:, 0], samples[:, axis_idx], 'b.')
    plt.plot(ovl_omega, ovl_adc, 'r')

    plt.show()

if __name__ == "__main__":
    main()


import numpy as np
from keras.optimizers import SGD
from models.CNN_CIFAR import cnn_cifar_batchnormalisation, cnn_cifar_small, cnn_cifar_nodropout, \
    cnn_cifar_small_batchnormalisation
from models.DataLoader import DataLoader
from scripts.progress_bar import printProgress
from time import time, localtime
# this is a predefined dataloader
loader = DataLoader(batch_size=32)

# construct the model here (pre-defined model)
model = cnn_cifar_small_batchnormalisation(loader.image_shape)
print(model.name)

nb_epoch = 200
early_stopping = True
early_stopping_count = 0
early_stopping_wait = 3
train_loss = []
valid_loss = []
learning_rate = [0.0001, 0.001, 0.01]

# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=learning_rate[-1], decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer=sgd)

# load validation data from the h5py file (heavy lifting here)
x_valid, y_valid = loader.get_valid()
best_valid = np.inf

for e in range(nb_epoch):
    print("epoch %d" % e)
    loss_list = []
    time_list = []
    time_start = time()
    for i in range(loader.n_iter_train):
        time_start_batch = time()
        X_batch, Y_batch = loader.next_train_batch()
        loss_list.append(model.train_on_batch(X_batch, Y_batch))

        # calculate some time information
        time_list.append(time() - time_start_batch)
        eta = (loader.n_iter_train - i) * np.array(time_list).mean()
        printProgress(i, loader.n_iter_train-1, prefix='Progress:', suffix='batch error: %0.5f, ETA: %0.2f sec.'%(np.array(loss_list).mean(), eta), barLength=50)
    printProgress(i, loader.n_iter_train - 1, prefix='Progress:', suffix='batch error: %0.5f' % (np.array(loss_list).mean()), barLength=50)
    train_loss.append(np.asarray(loss_list).mean())
    print('training loss is %f, one epoch uses: %0.2f sec' % (train_loss[-1], time() - time_start))
    valid_loss.append(model.evaluate(x_valid, y_valid))
    print('valid loss is %f' % valid_loss[-1])

    if best_valid > valid_loss[-1]:
        early_stopping_count = 0
        print('saving best valid result...')
        best_valid = valid_loss[-1]
        model.save('./models/CNN_Model_OBT100_multi_cnn_best_valid_'+model.name+'.h5')
    else:
        # we wait for early stopping loop until a certain time
        early_stopping_count += 1
        if early_stopping_count > early_stopping_wait:
            early_stopping_count = 0
            if len(learning_rate) > 1:
                learning_rate.pop()
                print('decreasing the learning rate to: %f'%learning_rate[-1])
                model.optimizer.lr.set_value(learning_rate[-1])
            else:
                break

lt = localtime()
lt_str = str(lt.tm_year)+"."+str(lt.tm_mon).zfill(2)+"." \
            +str(lt.tm_mday).zfill(2)+"."+str(lt.tm_hour).zfill(2)+"."\
            +str(lt.tm_min).zfill(2)+"."+str(lt.tm_sec).zfill(2)
np.savetxt('./models/train_loss_'+model.name+'_'+lt_str+'.txt', train_loss)
np.savetxt('./models/valid_loss_'+model.name+'_'+lt_str+'.txt', valid_loss)
model.save('./models/CNN_Model_OBT100_multi_cnn_'+model.name+'_final.h5')
print("done")

#### we show some visualisation here
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
train_loss = np.loadtxt('./models/train_loss_'+model.name+'_'+lt_str+'.txt')
valid_loss = np.loadtxt('./models/valid_loss_'+model.name+'_'+lt_str+'.txt')

plt.plot(train_loss, 'b')
plt.plot(valid_loss, 'r')

blue_label = mpatches.Patch(color='blue', label='train_loss')
red_label = mpatches.Patch(color='red', label='valid_loss')
plt.legend(handles=[blue_label, red_label])


'''
Created on Apr 28, 2013
Copyright: Jev Kuznetsov
License: BSD
'''
from __future__ import print_function
import sys
import urllib
import os
import xlrd # module for excel file reading
import pandas as pd

class ProgressBar:
    def __init__(self, iterations):
        self.iterations = iterations
        self.prog_bar = '[]'
        self.fill_char = '*'
        self.width = 50
        self.__update_amount(0)

    def animate(self, iteration):
        print('\r', self, end='')
        sys.stdout.flush()
        self.update_iteration(iteration + 1)

    def update_iteration(self, elapsed_iter):
        self.__update_amount((elapsed_iter / float(self.iterations)) * 100.0)
        self.prog_bar += '  %d of %s complete' % (elapsed_iter, self.iterations)

    def __update_amount(self, new_amount):
        percent_done = int(round((new_amount / 100.0) * 100.0))
        all_full = self.width - 2
        num_hashes = int(round((percent_done / 100.0) * all_full))
        self.prog_bar = '[' + self.fill_char * num_hashes + ' ' * (all_full - num_hashes) + ']'
        pct_place = (len(self.prog_bar) // 2) - len(str(percent_done))
        pct_string = '%d%%' % percent_done
        self.prog_bar = self.prog_bar[0:pct_place] + \
            (pct_string + self.prog_bar[pct_place + len(pct_string):])

    def __str__(self):
        return str(self.prog_bar)

def getSpyHoldings(dataDir):
    ''' get SPY holdings from the net, uses temp data storage to save xls file '''

    dest = os.path.join(dataDir,"spy_holdings.xls")

    if os.path.exists(dest):
        print('File found, skipping download')
    else:
        print('saving to', dest)
        urllib.urlretrieve ("https://www.spdrs.com/site-content/xls/SPY_All_Holdings.xls?fund=SPY&docname=All+Holdings&onyx_code1=1286&onyx_code2=1700",
                             dest) # download xls file and save it to data directory

    # parse
    wb = xlrd.open_workbook(dest) # open xls file, create a workbook
    sh = wb.sheet_by_index(0) # select first sheet

    data = {'name':[], 'symbol':[], 'weight':[],'sector':[]}
    for rowNr  in range(5,505): # cycle through the rows
        v = sh.row_values(rowNr) # get all row values
        data['name'].append(v[0])
        data['symbol'].append(v[1]) # symbol is in the second column, append it to the list
        data['weight'].append(float(v[2]))
        data['sector'].append(v[3])

    return  pd.DataFrame(data)


#PAM.py

import re
import glob, os, time
from numpy import *
from pylab import *

def analyzeFile(fileName,delim):
    cols = {}
    indexToName = {}
    lineNum = 0
    goodLines = 0
    shortLines = 0

    FILE = open(fileName,'r')

    for line in FILE:
        line = line.strip()

	if lineNum < 1:
	    lineNum += 1
	    continue
	elif lineNum == 1:
	   headings = line.split(delim)
	   i = 0
	   for heading in headings:
	       heading = heading.strip()
	       cols[heading] = []
	       indexToName[i] = heading
	       i += 1
	   lineNum += 1
	   lineLength = len(cols)
	else:
	   data = line.split(delim)
	   if len(data) == lineLength:
	       goodLines += 1
	       i = 0
	       for point in data:
	           point = point.strip()
	           cols[indexToName[i]] += [point]
	           i += 1
	       lineNum += 1
	   else:
	       shortLines += 1
	       lineNum += 1
	       continue
    FILE.close

    return cols, indexToName, lineNum, shortLines

def numericalSort(value):
	numbers = re.compile(r'(\d+)')
	parts = numbers.split(value)
	parts[1::2] = map(int, parts[1::2])
	return parts

def popDate(fileName):
	run = fileName.split('.')[0]
	runNo = run.split('_')[-1]
	return runNo

def getFile(date,regex):#Works
        files = []
	files = sorted((glob.glob('*'+regex+'*')),key=numericalSort,reverse=False)
	if date.lower() == 'last':
		files = files.pop()

	else:
	    files = [item for item in files if re.search(date,item)]

	return files

def plotConc(data,ozone,times):
        # This function plots data versus time
	import datetime as dt
	from matplotlib import pyplot as plt
	from matplotlib.dates import date2num

	#time = [dt.datetime.strptime(time,"%m/%d/%Y %I:%M:%S %p") for time in times]
        time = [dt.datetime.strptime(time,"%m/%d/%Y %I:%M:%S %p") for time in times]
	x = date2num(time)
        legend1 = []
	legend2 = []

	fig = plt.figure('Gas Concentration Readings for East St.Louis')
	ax1 = fig.add_subplot(111)
	ax2 = twinx()
	for key,value in data.items():
                ax1.plot_date(x,data[key],'-',xdate=True)
                legend1.append(key)
        for key, value in ozone.items():
            ax2.plot_date(x,ozone[key],'-.',xdate=True)
            legend2.append(key)

	title('Gas Concentrations for East St. Louis', fontsize = 12)
	ax1.set_ylabel(r'$Concentration(ppb)$', fontsize = 12)
	ax2.set_ylabel(r'$Concentration(ppb)$', fontsize = 12)
	xlabel(r"$Time \, Stamp$", fontsize = 12)

	ax1.legend(legend1,loc='upper right')
	ax2.legend(legend2,loc='lower right')

	grid(True)

	return

def plotBankRelays(data,relays,times):
        # This function plots data versus time
	import datetime as dt
	from matplotlib import pyplot as plt
	from matplotlib.dates import date2num

	time = [dt.datetime.strptime(time,"%m/%d/%Y %I:%M:%S %p") for time in times]
	x = date2num(time)
	#x1 = [date.strftime("%m-%d %H:%M:%S") for date in time]
        legend1 = []
        legend2 = []

	#plt.locator_params(axis='x', nbins=4)
	fig = plt.figure('VAPS Thermocouple Readings: Chart 2')
	ax1 = fig.add_subplot(111)
	ax2 = twinx()

	for key,value in data.items():
                ax1.plot_date(x,data[key],'-',xdate=True)
                legend1.append(key)
        for key,value in relays.items():
                ax2.plot_date(x,relays[key],'--',xdate=True)
                legend2.append(key)

	title('VAPS Temperatures: Chart 2', fontsize = 12)
	ax1.set_ylabel(r'$Temperature(^oC)$', fontsize = 12)
	ax2.set_ylabel(r'$Relay \, States$', fontsize = 12)
	ax1.set_xlabel(r"$Time \, Stamp$", fontsize = 12)
	#print [num2date(item) for item in ax1.get_xticks()]

	#ax1.set_xticks(x)

	#ax1.set_xticklabels([date.strftime("%m-%d %H:%M %p") for date in time])
        #ax1.legend(bbox_to_anchor=(0.,1.02,1.,.102),loc=3,ncol=2,mode="expand",borderaxespad=0.)
	ax1.legend(legend1,loc='upper right')
	ax2.legend(legend2,loc='lower right')
	#ax1.xaxis.set_major_formatter(FormatStrFormatter(date.strftime("%m-%d %H:%M:%S")))
	plt.subplots_adjust(bottom=0.15)
	grid(True)

	return

def goodFiles(files,goodHeaders,delim):   # Good
        irregFiles = 0
        goodFiles = []
        for file in files:
            lineNo = 0
            falseCount = 0
            FILE = open(file,'r')
            for line in FILE:
                line = line.strip()
                if lineNo == 5:
                    # Check all the headings to make sure the file is good
                    head = line.split(delim)
                    for item in head:
                        if item in goodHeaders:
                            continue
                        else:
                            falseCount += 1
                    if falseCount == 0:
                        goodFiles.append(file)
                    else:
                        irregFiles += 1
                    lineNo += 1

                else:
                    lineNo += 1
                    continue
            FILE.close
        return goodFiles, irregFiles


##                                COPYRIGHT
##    Copyright (C) 2015 Kyle Briggs (kbrig035<at>uottawa.ca)
##
##    This file is part of cusumtools.
##
##    This program is free software: you can redistribute it and/or modify
##    it under the terms of the GNU General Public License as published by
##    the Free Software Foundation, either version 3 of the License, or
##    (at your option) any later version.
##
##    This program is distributed in the hope that it will be useful,
##    but WITHOUT ANY WARRANTY; without even the implied warranty of
##    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##    GNU General Public License for more details.
##
##    You should have received a copy of the GNU General Public License
##    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import matplotlib
matplotlib.use('TkAgg')
import numpy as np
import tkinter.filedialog
import tkinter as tk
from matplotlib.figure import Figure
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg
import scipy.io as sio
from scipy.signal import bessel, filtfilt, welch
from scikits.samplerate import resample
import pylab as pl
import glob
import os
import time
import pandas as pd
from pandasql import sqldf
import re

def make_format(current, other):
    # current and other are axes
    def format_coord(x, y):
        # x, y are data coordinates
        # convert to display coords
        display_coord = current.transData.transform((x,y))
        inv = other.transData.inverted()
        # convert back to data coords with respect to ax
        ax_coord = inv.transform(display_coord)
        coords = [ax_coord, (x, y)]
        return ('Left: {:<40}    Right: {:<}'
                .format(*['({:.3f}, {:.3f})'.format(x, y) for x,y in coords]))
    return format_coord

class App(tk.Frame):
    def __init__(self, parent,file_path):
        tk.Frame.__init__(self, parent)
        parent.deiconify()
        self.events_flag = False
        self.baseline_flag = False
        self.file_path = file_path

        ##### Trace plotting widgets #####
        self.trace_frame = tk.LabelFrame(parent,text='Current Trace')
        self.trace_fig = Figure(figsize=(7,5), dpi=100)
        self.trace_canvas = FigureCanvasTkAgg(self.trace_fig, master=self.trace_frame)
        self.trace_toolbar_frame = tk.Frame(self.trace_frame)
        self.trace_toolbar = NavigationToolbar2TkAgg(self.trace_canvas, self.trace_toolbar_frame)
        self.trace_toolbar.update()

        self.trace_frame.grid(row=0,column=0,columnspan=6,sticky=tk.N+tk.S)
        self.trace_toolbar_frame.grid(row=1,column=0,columnspan=6)
        self.trace_canvas.get_tk_widget().grid(row=0,column=0,columnspan=6)

        ##### PSD plotting widgets #####
        self.psd_frame = tk.LabelFrame(parent,text='Power Spectrum')
        self.psd_fig = Figure(figsize=(7,5), dpi=100)
        self.psd_canvas = FigureCanvasTkAgg(self.psd_fig, master=self.psd_frame)
        self.psd_toolbar_frame = tk.Frame(self.psd_frame)
        self.psd_toolbar = NavigationToolbar2TkAgg(self.psd_canvas, self.psd_toolbar_frame)
        self.psd_toolbar.update()

        self.psd_frame.grid(row=0,column=6,columnspan=6,sticky=tk.N+tk.S)
        self.psd_toolbar_frame.grid(row=1,column=6,columnspan=6)
        self.psd_canvas.get_tk_widget().grid(row=0,column=6,columnspan=6)

        ##### Control widgets #####
        self.control_frame = tk.LabelFrame(parent, text='Controls')
        self.control_frame.grid(row=2,column=0,columnspan=6,sticky=tk.N+tk.S+tk.E+tk.W)

        self.start_entry = tk.Entry(self.control_frame)
        self.start_entry.insert(0,'0')
        self.start_label = tk.Label(self.control_frame, text='Start Time (s)')
        self.start_label.grid(row=0,column=0,sticky=tk.E+tk.W)
        self.start_entry.grid(row=0,column=1,sticky=tk.E+tk.W)

        self.end_entry = tk.Entry(self.control_frame)
        self.end_entry.insert(0,'10')
        self.end_label = tk.Label(self.control_frame, text='End Time (s)')
        self.end_label.grid(row=0,column=2,sticky=tk.E+tk.W)
        self.end_entry.grid(row=0,column=3,sticky=tk.E+tk.W)

        self.cutoff_entry = tk.Entry(self.control_frame)
        self.cutoff_entry.insert(0,'')
        self.cutoff_label = tk.Label(self.control_frame, text='Cutoff (Hz)')
        self.cutoff_label.grid(row=1,column=0,sticky=tk.E+tk.W)
        self.cutoff_entry.grid(row=1,column=1,sticky=tk.E+tk.W)

        self.order_entry = tk.Entry(self.control_frame)
        self.order_entry.insert(0,'')
        self.order_label = tk.Label(self.control_frame, text='Filter Order')
        self.order_label.grid(row=1,column=2,sticky=tk.E+tk.W)
        self.order_entry.grid(row=1,column=3,sticky=tk.E+tk.W)

        self.samplerate_entry = tk.Entry(self.control_frame)
        self.samplerate_entry.insert(0,'250000')
        self.samplerate_label = tk.Label(self.control_frame, text='Sampling Frequency (Hz)')
        self.samplerate_label.grid(row=1,column=4,sticky=tk.E+tk.W)
        self.samplerate_entry.grid(row=1,column=5,sticky=tk.E+tk.W)

        self.savegain_entry = tk.Entry(self.control_frame)
        self.savegain_entry.insert(0,'1')
        self.savegain_label = tk.Label(self.control_frame, text='Sampling Frequency (Hz)')
        self.savegain_label.grid(row=0,column=4,sticky=tk.E+tk.W)
        self.savegain_entry.grid(row=0,column=5,sticky=tk.E+tk.W)

        self.plot_trace = tk.Button(self.control_frame, text='Update Trace', command=self.update_trace)
        self.plot_trace.grid(row=2,column=0,columnspan=2,sticky=tk.E+tk.W)

        self.normalize = tk.IntVar()
        self.normalize.set(0)
        self.normalize_check = tk.Checkbutton(self.control_frame, text='Normalize', variable = self.normalize)
        self.normalize_check.grid(row=2,column=2,sticky=tk.E+tk.W)

        self.plot_psd = tk.Button(self.control_frame, text='Update PSD', command=self.update_psd)
        self.plot_psd.grid(row=2,column=3,sticky=tk.E+tk.W)

        ##### Feedback Widgets #####
        self.feedback_frame = tk.LabelFrame(parent, text='Status')
        self.feedback_frame.grid(row=2,column=6,columnspan=6,sticky=tk.N+tk.S+tk.E+tk.W)

        self.export_psd = tk.Button(self.feedback_frame, text='Export PSD',command=self.export_psd)
        self.export_psd.grid(row=1,column=0,columnspan=6,sticky=tk.E+tk.W)
        self.export_trace = tk.Button(self.feedback_frame, text='Export Trace',command=self.export_trace)
        self.export_trace.grid(row=2,column=0,columnspan=6,sticky=tk.E+tk.W)

        self.load_memmap()
        self.initialize_samplerate()

    def export_psd(self):
        try:
            data_path = tkinter.filedialog.asksaveasfilename(defaultextension='.csv',initialdir='G:\PSDs for Sam')
            np.savetxt(data_path,np.c_[self.f, self.Pxx, self.rms],delimiter=',')
        except AttributeError:
            self.wildcard.set('Plot the PSD first')

    def export_trace(self):
        try:
            data_path = tkinter.filedialog.asksaveasfilename(defaultextension='.csv',initialdir='G:\Analysis\Pores\NPN\PSDs')
            np.savetxt(data_path,self.plot_data,delimiter=',')
        except AttributeError:
            self.wildcard.set('Plot the trace first')

    def load_mapped_data(self):
        self.total_samples = len(self.map)
        self.samplerate = int(self.samplerate_entry.get())
        if self.start_entry.get()!='':
            self.start_time = float(self.start_entry.get())
            start_index = int((float(self.start_entry.get())*self.samplerate))
        else:
            self.start_time = 0
            start_index = 0

        if self.end_entry.get()!='':
            self.end_time = float(self.end_entry.get())
            end_index = int((float(self.end_entry.get())*self.samplerate))
            if end_index > self.total_samples:
                end_index = self.total_samples

        self.data = self.map[start_index:end_index]
        self.data = float(self.savegain_entry.get()) * self.data

    def load_memmap(self):
        columntypes = np.dtype([('current', '>i2'), ('voltage', '>i2')])
        self.map = np.memmap(self.file_path, dtype=columntypes, mode='r')['current']

    def integrate_noise(self, f, Pxx):
        df = f[1]-f[0]
        return np.sqrt(np.cumsum(Pxx * df))

    def filter_data(self):
        cutoff = float(self.cutoff_entry.get())
        order = int(self.order_entry.get())
        Wn = 2.0 * cutoff/float(self.samplerate)
        b, a = bessel(order,Wn,'low')
        padding = 1000
        padded = np.pad(self.data, pad_width=padding, mode='median')
        self.filtered_data = filtfilt(b, a, padded, padtype=None)[padding:-padding]

    def initialize_samplerate(self):
        self.samplerate = float(self.samplerate_entry.get())

    ##### Plot Updating functions #####
    def update_trace(self):
        self.initialize_samplerate()
        self.load_mapped_data()
        self.filtered_data = self.data
        self.plot_data = self.filtered_data
        plot_samplerate = self.samplerate

        if self.cutoff_entry.get()!='' and self.order_entry!='':
            self.filter_data()
            self.plot_data = self.filtered_data

        self.trace_fig.clf()
        a = self.trace_fig.add_subplot(111)

        time = np.linspace(1.0/self.samplerate,len(self.plot_data)/float(self.samplerate),len(self.plot_data))+self.start_time

        a.set_xlabel(r'Time ($\mu s$)')
        a.set_ylabel('Current (pA)')
        self.trace_fig.subplots_adjust(bottom=0.14,left=0.21)
        a.plot(time*1e6,self.plot_data,'.',markersize=1)

        self.trace_canvas.show()

    def update_psd(self):
        self.initialize_samplerate()
        self.load_mapped_data()
        self.filtered_data = self.data
        self.plot_data = self.filtered_data
        plot_samplerate = self.samplerate

        if self.cutoff_entry.get()!='' and self.order_entry!='':
            self.filter_data()
            self.plot_data = self.filtered_data
            maxf = 2*float(self.cutoff_entry.get())
        else:
            maxf = 2*float(self.samplerate_entry.get())

        length = np.minimum(2**18,len(self.filtered_data))
        end_index = int(np.floor(len(self.filtered_data)/length)*length)

        current = np.average(self.filtered_data[:end_index])

        f, Pxx = welch(self.filtered_data, plot_samplerate,nperseg=length)
        self.rms = self.integrate_noise(f, Pxx)

        if self.normalize.get():
            Pxx /= current**2
            Pxx *= maxf/2.0
            self.rms /= np.absolute(current)

        self.f = f
        self.Pxx = Pxx

        minf = 1
        BW_index = np.searchsorted(f, maxf/2)
        logPxx = np.log10(Pxx[1:BW_index])
        minP = 10**np.floor(np.amin(logPxx))
        maxP = 10**np.ceil(np.amax(logPxx))

        self.psd_fig.clf()
        a = self.psd_fig.add_subplot(111)
        a.set_xlabel('Frequency (Hz)')
        a.set_ylabel(r'Spectral Power ($\mathrm{pA}^2/\mathrm{Hz}$)')
        a.set_xlim(minf, maxf)
        a.set_ylim(minP, maxP)
        self.psd_fig.subplots_adjust(bottom=0.14,left=0.21)
        a.loglog(f[1:],Pxx[1:],'b-')
        for tick in a.get_yticklabels():
            tick.set_color('b')

        a2 = a.twinx()
        a2.semilogx(f, self.rms, 'r-')
        a2.set_ylabel('RMS Noise (pA)')
        a2.set_xlim(minf, maxf)
        for tick in a2.get_yticklabels():
            tick.set_color('r')
        a2.format_coord = make_format(a2, a)

        self.psd_canvas.show()

def main():
    root=tk.Tk()
    root.withdraw()
    file_path = tkinter.filedialog.askopenfilename(initialdir='C:/Data/')
    App(root,file_path).grid(row=0,column=0)
    root.mainloop()

if __name__=="__main__":
    main()


import os
import shutil

import numpy as np
import pandas as pd
import pytest

from .. import misc

class _FakeTable(object):
    def __init__(self, name, columns):
        self.name = name
        self.columns = columns

@pytest.fixture
def fta():
    return _FakeTable('a', ['aa', 'ab', 'ac'])

@pytest.fixture
def ftb():
    return _FakeTable('b', ['bx', 'by', 'bz'])

@pytest.fixture
def clean_fake_data_home(request):
    def fin():
        if os.path.isdir('fake_data_home'):
            shutil.rmtree('fake_data_home')
    request.addfinalizer(fin)

def test_column_map_raises(fta, ftb):
    with pytest.raises(RuntimeError):
        misc.column_map([fta, ftb], ['aa', 'by', 'bz', 'cw'])

def test_column_map_none(fta, ftb):
    assert misc.column_map([fta, ftb], None) == {'a': None, 'b': None}

def test_column_map(fta, ftb):
    assert misc.column_map([fta, ftb], ['aa', 'by', 'bz']) == \
        {'a': ['aa'], 'b': ['by', 'bz']}
    assert misc.column_map([fta, ftb], ['by', 'bz']) == \
        {'a': [], 'b': ['by', 'bz']}

def test_dirs(clean_fake_data_home):
    misc._mkifnotexists("fake_data_home")
    os.environ["DATA_HOME"] = "fake_data_home"
    misc.get_run_number()
    misc.get_run_number()
    misc.data_dir()
    misc.configs_dir()
    misc.models_dir()
    misc.charts_dir()
    misc.maps_dir()
    misc.simulations_dir()
    misc.reports_dir()
    misc.runs_dir()
    misc.config("test")

@pytest.fixture
def range_df():
    df = pd.DataFrame({'to_zone_id': [2, 3, 4],
                       'from_zone_id': [1, 1, 1],
                       'distance': [.1, .2, .9]})
    df = df.set_index(['from_zone_id', 'to_zone_id'])
    return df

@pytest.fixture
def range_series():
    return pd.Series([10, 150, 75, 275], index=[1, 2, 3, 4])

def test_compute_range(range_df, range_series):
    assert misc.compute_range(range_df, range_series, "distance", .5).loc[1] == 225

def test_reindex():
    s = pd.Series([.5, 1.0, 1.5], index=[2, 1, 3])
    s2 = pd.Series([1, 2, 3], index=['a', 'b', 'c'])
    assert list(misc.reindex(s, s2).values) == [1.0, .5, 1.5]

def test_naics():
    assert misc.naicsname(54) == "Professional"

def test_signif():
    assert misc.signif(4.0) == '***'
    assert misc.signif(3.0) == '**'
    assert misc.signif(2.0) == '*'
    assert misc.signif(1.5) == '.'
    assert misc.signif(1.0) == ''

@pytest.fixture
def simple_dev_inputs():
    return pd.DataFrame(
        {'residential': [40, 40, 40],
         'office': [15, 18, 15],
         'retail': [12, 10, 10],
         'industrial': [12, 12, 12],
         'land_cost': [1000000, 2000000, 3000000],
         'parcel_size': [10000, 20000, 30000],
         'max_far': [2.0, 3.0, 4.0],
         'names': ['a', 'b', 'c'],
         'max_height': [40, 60, 80]},
        index=['a', 'b', 'c'])

def test_misc_dffunctions(simple_dev_inputs):
    misc.df64bitto32bit(simple_dev_inputs)
    misc.pandasdfsummarytojson(simple_dev_inputs[['land_cost', 'parcel_size']])
    misc.numpymat2df(np.array([[1, 2], [3, 4]]))

def test_column_list(fta, ftb):
    assert misc.column_list([fta, ftb], ['aa', 'by', 'bz', 'c']) == \
        ['aa', 'by', 'bz']


# -*- coding: utf-8 -*-
"""
A collection of widgets for gui building

Copyright: Jev Kuznetsov
License: BSD
"""

from __future__ import division

import sys
from PyQt4.QtCore import *
from PyQt4.QtGui import *

import numpy as np
from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt4agg import NavigationToolbar2QTAgg as NavigationToolbar
from matplotlib.figure import Figure
import matplotlib.pyplot as plt

class MatplotlibWidget(QWidget):
    def __init__(self,parent=None,grid=True):
        QWidget.__init__(self,parent)

        self.grid = grid

        self.fig = Figure()
        self.canvas =FigureCanvas(self.fig)
        self.canvas.setParent(self)
        self.canvas.mpl_connect('button_press_event', self.onPick) # bind pick event

        #self.axes = self.fig.add_subplot(111)
        margins = [0.05,0.1,0.9,0.8]
        self.axes = self.fig.add_axes(margins)
        self.toolbar = NavigationToolbar(self.canvas,self)

        #self.initFigure()

        layout = QVBoxLayout()
        layout.addWidget(self.toolbar)
        layout.addWidget(self.canvas)

        self.setLayout(layout)

    def onPick(self,event):
        print 'Pick event'
        print 'you pressed', event.button, event.xdata, event.ydata

    def update(self):
        self.canvas.draw()

    def plot(self,*args,**kwargs):
        self.axes.plot(*args,**kwargs)
        self.axes.grid(self.grid)
        self.update()

    def clear(self):
        self.axes.clear()

    def initFigure(self):
        self.axes.grid(True)
        x = np.linspace(-1,1)
        y = x**2
        self.axes.plot(x,y,'o-')

class PlotWindow(QMainWindow):
    ''' a stand-alone window with embedded matplotlib widget '''
    def __init__(self,parent=None):
        super(PlotWindow,self).__init__(parent)
        self.setAttribute(Qt.WA_DeleteOnClose)
        self.mplWidget = MatplotlibWidget()
        self.setCentralWidget(self.mplWidget)

    def plot(self,dataFrame):
        ''' plot dataframe '''
        dataFrame.plot(ax=self.mplWidget.axes)

    def getAxes(self):
        return self.mplWidget.axes

    def getFigure(self):
        return self.mplWidget.fig

    def update(self):
        self.mplWidget.update()

class MainForm(QMainWindow):
    def __init__(self, parent=None):
        QMainWindow.__init__(self, parent)
        self.setWindowTitle('Demo: PyQt with matplotlib')

        self.plot = MatplotlibWidget()
        self.setCentralWidget(self.plot)

        self.plot.clear()
        self.plot.plot(np.random.rand(10),'x-')

#---------------------
if __name__=='__main__':
    app = QApplication(sys.argv)
    form = MainForm()
    form.show()
    app.exec_()


import pandas as pd
import numpy as np

def ss_calc(
    contrib_yearly, inv_gwth_rt, num_years, safe_withdrw_rate, start_age=28
):
    """
    inv_gwth_rt is infaltion adjusted.
    contrib_yearly is in first years dollars
    """
    tot_years = max(0, 62 - start_age - num_years) + num_years
    df = pd.DataFrame({
        'contrib_yearly': [contrib_yearly] * num_years + [0.] *
        max(0, (62 - num_years - start_age)),
        'inv_value': [0] * tot_years,
    }, index=range(tot_years))
    for year in range(0, tot_years):
        print year
        multiplier = np.array([
            (1. + inv_gwth_rt) ** (year - y_) for y_ in range(year + 1)])
        print multiplier
        df['inv_value'][year] = np.sum(
            np.array(df['contrib_yearly'][0: year + 1]) * multiplier)
    df['monthly_inv_income'] = safe_withdrw_rate * df['inv_value'] / 12.
    df['monthly_inv_income_w_spouse'] = df['monthly_inv_income'] * 1.5
    return df

if __name__ == "__main__":
    df = ss_calc(15e3, .03, 10, .03)

    ss_benefit_monthly = 939.00
    ss_benefit_w_spouse_monthly = 1.5 * ss_benefit_monthly