import numpy as np
try:
  import matplotlib.pyplot as plt
except ImportError:
  plt = None
from numpy.testing import assert_equal, assert_raises

from skimage.feature.util import (FeatureDetector, DescriptorExtractor,
                                  _prepare_grayscale_input_2D,
                                  _mask_border_keypoints, plot_matches)

def test_feature_detector():
    assert_raises(NotImplementedError, FeatureDetector().detect, None)

def test_descriptor_extractor():
    assert_raises(NotImplementedError, DescriptorExtractor().extract,
                  None, None)

def test_prepare_grayscale_input_2D():
    assert_raises(ValueError, _prepare_grayscale_input_2D, np.zeros((3, 3, 3)))
    assert_raises(ValueError, _prepare_grayscale_input_2D, np.zeros((3, 1)))
    assert_raises(ValueError, _prepare_grayscale_input_2D, np.zeros((3, 1, 1)))
    img = _prepare_grayscale_input_2D(np.zeros((3, 3)))
    img = _prepare_grayscale_input_2D(np.zeros((3, 3, 1)))
    img = _prepare_grayscale_input_2D(np.zeros((1, 3, 3)))

def test_mask_border_keypoints():
    keypoints = np.array([[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]])
    assert_equal(_mask_border_keypoints((10, 10), keypoints, 0),
                 [1, 1, 1, 1, 1])
    assert_equal(_mask_border_keypoints((10, 10), keypoints, 2),
                 [0, 0, 1, 1, 1])
    assert_equal(_mask_border_keypoints((4, 4), keypoints, 2),
                 [0, 0, 1, 0, 0])
    assert_equal(_mask_border_keypoints((10, 10), keypoints, 5),
                 [0, 0, 0, 0, 0])
    assert_equal(_mask_border_keypoints((10, 10), keypoints, 4),
                 [0, 0, 0, 0, 1])

@np.testing.decorators.skipif(plt is None)
def test_plot_matches():
    fig, ax = plt.subplots(nrows=1, ncols=1)

    shapes = (((10, 10), (10, 10)),
              ((10, 10), (12, 10)),
              ((10, 10), (10, 12)),
              ((10, 10), (12, 12)),
              ((12, 10), (10, 10)),
              ((10, 12), (10, 10)),
              ((12, 12), (10, 10)))

    keypoints1 = 10 * np.random.rand(10, 2)
    keypoints2 = 10 * np.random.rand(10, 2)
    idxs1 = np.random.randint(10, size=10)
    idxs2 = np.random.randint(10, size=10)
    matches = np.column_stack((idxs1, idxs2))

    for shape1, shape2 in shapes:
        img1 = np.zeros(shape1)
        img2 = np.zeros(shape2)
        plot_matches(ax, img1, img2, keypoints1, keypoints2, matches)
        plot_matches(ax, img1, img2, keypoints1, keypoints2, matches,
                     only_matches=True)
        plot_matches(ax, img1, img2, keypoints1, keypoints2, matches,
                     keypoints_color='r')
        plot_matches(ax, img1, img2, keypoints1, keypoints2, matches,
                     matches_color='r')

if __name__ == '__main__':
    from numpy import testing
    testing.run_module_suite()


"""
=============================================================
Receiver Operating Characteristic (ROC) with cross validation
=============================================================

Example of Receiver Operating Characteristic (ROC) metric to evaluate
classifier output quality using cross-validation.

ROC curves typically feature true positive rate on the Y axis, and false
positive rate on the X axis. This means that the top left corner of the plot is
the "ideal" point - a false positive rate of zero, and a true positive rate of
one. This is not very realistic, but it does mean that a larger area under the
curve (AUC) is usually better.

The "steepness" of ROC curves is also important, since it is ideal to maximize
the true positive rate while minimizing the false positive rate.

This example shows the ROC response of different datasets, created from K-fold
cross-validation. Taking all of these curves, it is possible to calculate the
mean area under curve, and see the variance of the curve when the
training set is split into different subsets. This roughly shows how the
classifier output is affected by changes in the training data, and how
different the splits generated by K-fold cross-validation are from one another.

.. note::

    See also :func:`sklearn.metrics.auc_score`,
             :func:`sklearn.model_selection.cross_val_score`,
             :ref:`sphx_glr_auto_examples_model_selection_plot_roc.py`,

"""
print(__doc__)

import numpy as np
from scipy import interp
import matplotlib.pyplot as plt
from itertools import cycle

from sklearn import svm, datasets
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import StratifiedKFold

###############################################################################
# Data IO and generation

# import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target
X, y = X[y != 2], y[y != 2]
n_samples, n_features = X.shape

# Add noisy features
random_state = np.random.RandomState(0)
X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]

###############################################################################
# Classification and ROC analysis

# Run classifier with cross-validation and plot ROC curves
cv = StratifiedKFold(n_splits=6)
classifier = svm.SVC(kernel='linear', probability=True,
                     random_state=random_state)

mean_tpr = 0.0
mean_fpr = np.linspace(0, 1, 100)

colors = cycle(['cyan', 'indigo', 'seagreen', 'yellow', 'blue', 'darkorange'])
lw = 2

i = 0
for (train, test), color in zip(cv.split(X, y), colors):
    probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test])
    # Compute ROC curve and area the curve
    fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
    mean_tpr += interp(mean_fpr, fpr, tpr)
    mean_tpr[0] = 0.0
    roc_auc = auc(fpr, tpr)
    plt.plot(fpr, tpr, lw=lw, color=color,
             label='ROC fold %d (area = %0.2f)' % (i, roc_auc))

    i += 1
plt.plot([0, 1], [0, 1], linestyle='--', lw=lw, color='k',
         label='Luck')

mean_tpr /= cv.get_n_splits(X, y)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
plt.plot(mean_fpr, mean_tpr, color='g', linestyle='--',
         label='Mean ROC (area = %0.2f)' % mean_auc, lw=lw)

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()


import numpy as np
import cv2
import time
import matplotlib.pylab as plt

"""
Make sure that you hold the checkerboard horizontally (more checkers horizontally than vertically).

In order to get a good calibration you will need to move the checkerboard around in the camera frame such that:

    the checkerboard is detected at the left and right edges of the field of view (X calibration)
    the checkerboard is detected at the top and bottom edges of the field of view (Y calibration)
    the checkerboard is detected at various angles to the camera ("Skew")
    the checkerboard fills the entire field of view (Size calibration)
    checkerboard tilted to the left, right, top and bottom (X,Y, and Size calibration)
"""

left = 1
right = 2

time_in_ms= 1000/100
#folder = "/home/cwu/Downloads/";
folder = "/home/hillcrest/project/stereo-calibration/calib_imgs/ARC/"

folder = "/home/hillcrest/project/stereo-calibration/calib_imgs/ARC/"
#folder = "D:/vision/stereo-calibration/calib_imgs/ARC/"

fp = open(folder + "timeStamp.txt","w")

WIDTH = 1280
HEIGHT = 720

WIDTH = 640
HEIGHT = 480

for counter in range(1,31):

    millis = int(round(time.time() * 1000))
    cap1 = cv2.VideoCapture(left)
    cap1.set(cv2.CAP_PROP_FRAME_WIDTH,WIDTH)
    cap1.set(cv2.CAP_PROP_FRAME_HEIGHT,HEIGHT)
    cv2.waitKey(100)
    ret, frame1 = cap1.read()

    cap1.release()

    cap2 = cv2.VideoCapture(right)
    cap2.set(cv2.CAP_PROP_FRAME_WIDTH,WIDTH)
    cap2.set(cv2.CAP_PROP_FRAME_HEIGHT,HEIGHT)
    cv2.waitKey(100)
    ret, frame2 = cap2.read()
    cap2.release()

    #frame1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
    #frame2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
    # Display the resulting frame

    plt.subplot(121)
    plt.imshow(frame1)
    plt.title('left')

    plt.subplot(122)
    plt.imshow(frame2)
    plt.title('right')

    plt.show()
    print('another capture', counter)
    cv2.waitKey(100)

    cv2.imwrite(folder + "img_left/left_" + str(counter) + ".jpg",  frame1)
    cv2.waitKey(time_in_ms)
    cv2.imwrite(folder + "img_right/right_" + str(counter) + ".jpg", frame2)

    fp.write(str(counter)+ ","+ str(millis) + "\n")
    print("the ", counter, " pairs")

cv2.destroyAllWindows()

fp.close()
print('All Done \n')


"""
Testing Recursive feature elimination
"""

from operator import attrgetter
import pytest
import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal
from scipy import sparse

from sklearn.feature_selection import RFE, RFECV
from sklearn.datasets import load_iris, make_friedman1
from sklearn.metrics import zero_one_loss
from sklearn.svm import SVC, SVR, LinearSVR
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GroupKFold
from sklearn.compose import TransformedTargetRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

from sklearn.utils import check_random_state
from sklearn.utils._testing import ignore_warnings

from sklearn.metrics import make_scorer
from sklearn.metrics import get_scorer

class MockClassifier:
    """
    Dummy classifier to test recursive feature elimination
    """

    def __init__(self, foo_param=0):
        self.foo_param = foo_param

    def fit(self, X, y):
        assert len(X) == len(y)
        self.coef_ = np.ones(X.shape[1], dtype=np.float64)
        return self

    def predict(self, T):
        return T.shape[0]

    predict_proba = predict
    decision_function = predict
    transform = predict

    def score(self, X=None, y=None):
        return 0.

    def get_params(self, deep=True):
        return {'foo_param': self.foo_param}

    def set_params(self, **params):
        return self

    def _more_tags(self):
        return {"allow_nan": True}

def test_rfe_features_importance():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    clf = RandomForestClassifier(n_estimators=20,
                                 random_state=generator, max_depth=2)
    rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    assert len(rfe.ranking_) == X.shape[1]

    clf_svc = SVC(kernel="linear")
    rfe_svc = RFE(estimator=clf_svc, n_features_to_select=4, step=0.1)
    rfe_svc.fit(X, y)

    # Check if the supports are equal
    assert_array_equal(rfe.get_support(), rfe_svc.get_support())

def test_rfe():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    X_sparse = sparse.csr_matrix(X)
    y = iris.target

    # dense model
    clf = SVC(kernel="linear")
    rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    X_r = rfe.transform(X)
    clf.fit(X_r, y)
    assert len(rfe.ranking_) == X.shape[1]

    # sparse model
    clf_sparse = SVC(kernel="linear")
    rfe_sparse = RFE(estimator=clf_sparse, n_features_to_select=4, step=0.1)
    rfe_sparse.fit(X_sparse, y)
    X_r_sparse = rfe_sparse.transform(X_sparse)

    assert X_r.shape == iris.data.shape
    assert_array_almost_equal(X_r[:10], iris.data[:10])

    assert_array_almost_equal(rfe.predict(X), clf.predict(iris.data))
    assert rfe.score(X, y) == clf.score(iris.data, iris.target)
    assert_array_almost_equal(X_r, X_r_sparse.toarray())

@pytest.mark.parametrize("n_features_to_select", [-1, 2.1])
def test_rfe_invalid_n_features_errors(n_features_to_select):
    clf = SVC(kernel="linear")

    iris = load_iris()
    rfe = RFE(estimator=clf, n_features_to_select=n_features_to_select,
              step=0.1)
    msg = f"n_features_to_select must be .+ Got {n_features_to_select}"
    with pytest.raises(ValueError, match=msg):
        rfe.fit(iris.data, iris.target)

def test_rfe_percent_n_features():
    # test that the results are the same
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target
    # there are 10 features in the data. We select 40%.
    clf = SVC(kernel="linear")
    rfe_num = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe_num.fit(X, y)

    rfe_perc = RFE(estimator=clf, n_features_to_select=0.4, step=0.1)
    rfe_perc.fit(X, y)

    assert_array_equal(rfe_perc.ranking_, rfe_num.ranking_)
    assert_array_equal(rfe_perc.support_, rfe_num.support_)

def test_rfe_mockclassifier():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    # dense model
    clf = MockClassifier()
    rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    X_r = rfe.transform(X)
    clf.fit(X_r, y)
    assert len(rfe.ranking_) == X.shape[1]
    assert X_r.shape == iris.data.shape

def test_rfecv():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = list(iris.target)   # regression test: list should be supported

    # Test using the score function
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1)
    rfecv.fit(X, y)
    # non-regression test for missing worst feature:
    assert len(rfecv.grid_scores_) == X.shape[1]
    assert len(rfecv.ranking_) == X.shape[1]
    X_r = rfecv.transform(X)

    # All the noisy variable were filtered out
    assert_array_equal(X_r, iris.data)

    # same in sparse
    rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=1)
    X_sparse = sparse.csr_matrix(X)
    rfecv_sparse.fit(X_sparse, y)
    X_r_sparse = rfecv_sparse.transform(X_sparse)
    assert_array_equal(X_r_sparse.toarray(), iris.data)

    # Test using a customized loss function
    scoring = make_scorer(zero_one_loss, greater_is_better=False)
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, scoring=scoring)
    ignore_warnings(rfecv.fit)(X, y)
    X_r = rfecv.transform(X)
    assert_array_equal(X_r, iris.data)

    # Test using a scorer
    scorer = get_scorer('accuracy')
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, scoring=scorer)
    rfecv.fit(X, y)
    X_r = rfecv.transform(X)
    assert_array_equal(X_r, iris.data)

    # Test fix on grid_scores
    def test_scorer(estimator, X, y):
        return 1.0
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, scoring=test_scorer)
    rfecv.fit(X, y)
    assert_array_equal(rfecv.grid_scores_, np.ones(len(rfecv.grid_scores_)))
    # In the event of cross validation score ties, the expected behavior of
    # RFECV is to return the FEWEST features that maximize the CV score.
    # Because test_scorer always returns 1.0 in this example, RFECV should
    # reduce the dimensionality to a single feature (i.e. n_features_ = 1)
    assert rfecv.n_features_ == 1

    # Same as the first two tests, but with step=2
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=2)
    rfecv.fit(X, y)
    assert len(rfecv.grid_scores_) == 6
    assert len(rfecv.ranking_) == X.shape[1]
    X_r = rfecv.transform(X)
    assert_array_equal(X_r, iris.data)

    rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=2)
    X_sparse = sparse.csr_matrix(X)
    rfecv_sparse.fit(X_sparse, y)
    X_r_sparse = rfecv_sparse.transform(X_sparse)
    assert_array_equal(X_r_sparse.toarray(), iris.data)

    # Verifying that steps < 1 don't blow up.
    rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=.2)
    X_sparse = sparse.csr_matrix(X)
    rfecv_sparse.fit(X_sparse, y)
    X_r_sparse = rfecv_sparse.transform(X_sparse)
    assert_array_equal(X_r_sparse.toarray(), iris.data)

def test_rfecv_mockclassifier():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = list(iris.target)   # regression test: list should be supported

    # Test using the score function
    rfecv = RFECV(estimator=MockClassifier(), step=1)
    rfecv.fit(X, y)
    # non-regression test for missing worst feature:
    assert len(rfecv.grid_scores_) == X.shape[1]
    assert len(rfecv.ranking_) == X.shape[1]

def test_rfecv_verbose_output():
    # Check verbose=1 is producing an output.
    from io import StringIO
    import sys
    sys.stdout = StringIO()

    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = list(iris.target)

    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, verbose=1)
    rfecv.fit(X, y)

    verbose_output = sys.stdout
    verbose_output.seek(0)
    assert len(verbose_output.readline()) > 0

def test_rfecv_grid_scores_size():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = list(iris.target)   # regression test: list should be supported

    # Non-regression test for varying combinations of step and
    # min_features_to_select.
    for step, min_features_to_select in [[2, 1], [2, 2], [3, 3]]:
        rfecv = RFECV(estimator=MockClassifier(), step=step,
                      min_features_to_select=min_features_to_select)
        rfecv.fit(X, y)

        score_len = np.ceil(
            (X.shape[1] - min_features_to_select) / step) + 1
        assert len(rfecv.grid_scores_) == score_len
        assert len(rfecv.ranking_) == X.shape[1]
        assert rfecv.n_features_ >= min_features_to_select

def test_rfe_estimator_tags():
    rfe = RFE(SVC(kernel='linear'))
    assert rfe._estimator_type == "classifier"
    # make sure that cross-validation is stratified
    iris = load_iris()
    score = cross_val_score(rfe, iris.data, iris.target)
    assert score.min() > .7

def test_rfe_min_step():
    n_features = 10
    X, y = make_friedman1(n_samples=50, n_features=n_features, random_state=0)
    n_samples, n_features = X.shape
    estimator = SVR(kernel="linear")

    # Test when floor(step * n_features) <= 0
    selector = RFE(estimator, step=0.01)
    sel = selector.fit(X, y)
    assert sel.support_.sum() == n_features // 2

    # Test when step is between (0,1) and floor(step * n_features) > 0
    selector = RFE(estimator, step=0.20)
    sel = selector.fit(X, y)
    assert sel.support_.sum() == n_features // 2

    # Test when step is an integer
    selector = RFE(estimator, step=5)
    sel = selector.fit(X, y)
    assert sel.support_.sum() == n_features // 2

def test_number_of_subsets_of_features():
    # In RFE, 'number_of_subsets_of_features'
    # = the number of iterations in '_fit'
    # = max(ranking_)
    # = 1 + (n_features + step - n_features_to_select - 1) // step
    # After optimization #4534, this number
    # = 1 + np.ceil((n_features - n_features_to_select) / float(step))
    # This test case is to test their equivalence, refer to #4534 and #3824

    def formula1(n_features, n_features_to_select, step):
        return 1 + ((n_features + step - n_features_to_select - 1) // step)

    def formula2(n_features, n_features_to_select, step):
        return 1 + np.ceil((n_features - n_features_to_select) / float(step))

    # RFE
    # Case 1, n_features - n_features_to_select is divisible by step
    # Case 2, n_features - n_features_to_select is not divisible by step
    n_features_list = [11, 11]
    n_features_to_select_list = [3, 3]
    step_list = [2, 3]
    for n_features, n_features_to_select, step in zip(
            n_features_list, n_features_to_select_list, step_list):
        generator = check_random_state(43)
        X = generator.normal(size=(100, n_features))
        y = generator.rand(100).round()
        rfe = RFE(estimator=SVC(kernel="linear"),
                  n_features_to_select=n_features_to_select, step=step)
        rfe.fit(X, y)
        # this number also equals to the maximum of ranking_
        assert (np.max(rfe.ranking_) ==
                formula1(n_features, n_features_to_select, step))
        assert (np.max(rfe.ranking_) ==
                formula2(n_features, n_features_to_select, step))

    # In RFECV, 'fit' calls 'RFE._fit'
    # 'number_of_subsets_of_features' of RFE
    # = the size of 'grid_scores' of RFECV
    # = the number of iterations of the for loop before optimization #4534

    # RFECV, n_features_to_select = 1
    # Case 1, n_features - 1 is divisible by step
    # Case 2, n_features - 1 is not divisible by step

    n_features_to_select = 1
    n_features_list = [11, 10]
    step_list = [2, 2]
    for n_features, step in zip(n_features_list, step_list):
        generator = check_random_state(43)
        X = generator.normal(size=(100, n_features))
        y = generator.rand(100).round()
        rfecv = RFECV(estimator=SVC(kernel="linear"), step=step)
        rfecv.fit(X, y)

        assert (rfecv.grid_scores_.shape[0] ==
                formula1(n_features, n_features_to_select, step))
        assert (rfecv.grid_scores_.shape[0] ==
                formula2(n_features, n_features_to_select, step))

def test_rfe_cv_n_jobs():
    generator = check_random_state(0)
    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    rfecv = RFECV(estimator=SVC(kernel='linear'))
    rfecv.fit(X, y)
    rfecv_ranking = rfecv.ranking_
    rfecv_grid_scores = rfecv.grid_scores_

    rfecv.set_params(n_jobs=2)
    rfecv.fit(X, y)
    assert_array_almost_equal(rfecv.ranking_, rfecv_ranking)
    assert_array_almost_equal(rfecv.grid_scores_, rfecv_grid_scores)

def test_rfe_cv_groups():
    generator = check_random_state(0)
    iris = load_iris()
    number_groups = 4
    groups = np.floor(np.linspace(0, number_groups, len(iris.target)))
    X = iris.data
    y = (iris.target > 0).astype(int)

    est_groups = RFECV(
        estimator=RandomForestClassifier(random_state=generator),
        step=1,
        scoring='accuracy',
        cv=GroupKFold(n_splits=2)
    )
    est_groups.fit(X, y, groups=groups)
    assert est_groups.n_features_ > 0

@pytest.mark.parametrize(
    'importance_getter',
    [attrgetter('regressor_.coef_'), 'regressor_.coef_'])
@pytest.mark.parametrize('selector, expected_n_features',
                         [(RFE, 5), (RFECV, 4)])
def test_rfe_wrapped_estimator(importance_getter, selector,
                               expected_n_features):
    # Non-regression test for
    # https://github.com/scikit-learn/scikit-learn/issues/15312
    X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
    estimator = LinearSVR(random_state=0)

    log_estimator = TransformedTargetRegressor(regressor=estimator,
                                               func=np.log,
                                               inverse_func=np.exp)

    selector = selector(log_estimator, importance_getter=importance_getter)
    sel = selector.fit(X, y)
    assert sel.support_.sum() == expected_n_features

@pytest.mark.parametrize(
    "importance_getter, err_type",
    [("auto", ValueError),
     ("random", AttributeError),
     (lambda x: x.importance, AttributeError),
     ([0], ValueError)]
)
@pytest.mark.parametrize("Selector", [RFE, RFECV])
def test_rfe_importance_getter_validation(importance_getter, err_type,
                                          Selector):
    X, y = make_friedman1(n_samples=50, n_features=10, random_state=42)
    estimator = LinearSVR()
    log_estimator = TransformedTargetRegressor(
        regressor=estimator, func=np.log, inverse_func=np.exp
    )

    with pytest.raises(err_type):
        model = Selector(log_estimator, importance_getter=importance_getter)
        model.fit(X, y)

@pytest.mark.parametrize("cv", [None, 5])
def test_rfe_allow_nan_inf_in_x(cv):
    iris = load_iris()
    X = iris.data
    y = iris.target

    # add nan and inf value to X
    X[0][0] = np.NaN
    X[0][1] = np.Inf

    clf = MockClassifier()
    if cv is not None:
        rfe = RFECV(estimator=clf, cv=cv)
    else:
        rfe = RFE(estimator=clf)
    rfe.fit(X, y)
    rfe.transform(X)

def test_w_pipeline_2d_coef_():
    pipeline = make_pipeline(StandardScaler(), LogisticRegression())

    data, y = load_iris(return_X_y=True)
    sfm = RFE(pipeline, n_features_to_select=2,
              importance_getter='named_steps.logisticregression.coef_')

    sfm.fit(data, y)
    assert sfm.transform(data).shape[1] == 2

@pytest.mark.parametrize('ClsRFE', [
    RFE,
    RFECV
    ])
def test_multioutput(ClsRFE):
    X = np.random.normal(size=(10, 3))
    y = np.random.randint(2, size=(10, 2))
    clf = RandomForestClassifier(n_estimators=5)
    rfe_test = ClsRFE(clf)
    rfe_test.fit(X, y)


from django.contrib.auth.models import User
from mangaki.models import Rating, Work, Recommendation
from mangaki.utils.chrono import Chrono
from mangaki.utils.values import rating_values
from scipy.sparse import lil_matrix
from sklearn.utils.extmath import randomized_svd
import numpy as np
from django.db import connection
import pickle
import json
import math

NB_COMPONENTS = 10
TOP = 10

class MangakiSVD(object):
    M = None
    U = None
    sigma = None
    VT = None
    chrono = None
    inv_work = None
    inv_user = None
    work_titles = None
    def __init__(self):
        self.chrono = Chrono(True)

    def save(self, filename):
        with open(filename, 'wb') as f:
            pickle.dump(self, f)

    def load(self, filename):
        with open(filename, 'rb') as f:
            backup = pickle.load(f)
        self.M = backup.M
        self.U = backup.U
        self.sigma = backup.sigma
        self.VT = backup.VT
        self.inv_work = backup.inv_work
        self.inv_user = backup.inv_user
        self.work_titles = backup.work_titles

    def fit(self, X, y):
        self.work_titles = {}
        for work in Work.objects.values('id', 'title'):
            self.work_titles[work['id']] = work['title']

        work_ids = list(Rating.objects.values_list('work_id', flat=True).distinct())
        nb_works = len(work_ids)
        self.inv_work = {work_ids[i]: i for i in range(nb_works)}

        user_ids = list(User.objects.values_list('id', flat=True))
        nb_users = len(user_ids)
        self.inv_user = {user_ids[i]: i for i in range(nb_users)}

        self.chrono.save('get_work_ids')

        # print("Computing M: (%i × %i)" % (nb_users, nb_works))
        self.M = lil_matrix((nb_users, nb_works))
        """ratings_of = {}
        for (user_id, work_id), rating in zip(X, y):
            ratings_of.setdefault(user_id, []).append(rating)"""
        for (user_id, work_id), rating in zip(X, y):
            self.M[self.inv_user[user_id], self.inv_work[work_id]] = rating #- np.mean(ratings_of[user_id])
        # np.save('backupM', self.M)

        self.chrono.save('fill matrix')

        # Ranking computation
        self.U, self.sigma, self.VT = randomized_svd(self.M, NB_COMPONENTS, n_iter=3, random_state=42)
        # print('Formes', self.U.shape, self.sigma.shape, self.VT.shape)

        self.save('backup.pickle')

        self.chrono.save('factor matrix')

    def predict(self, X):
        y = []
        for user_id, work_id in X:
            i = self.inv_user[user_id]
            j = self.inv_work[work_id]
            y.append(self.U[i].dot(np.diag(self.sigma)).dot(self.VT.transpose()[j]))
        return np.array(y)

    def get_reco(self, username, sending=False):
        target_user = User.objects.get(username=username)
        the_user_id = target_user.id
        svd_user = User.objects.get(username='svd')

        work_ids = {self.inv_work[work_id]: work_id for work_id in self.inv_work}
        nb_works = len(work_ids)

        seen_works = set(Rating.objects.filter(user__id=the_user_id).exclude(choice='willsee').values_list('work_id', flat=True))
        the_i = self.inv_user[the_user_id]

        self.chrono.save('get_seen_works')

        print('mon vecteur (taille %d)' % len(self.U[the_i]), self.U[the_i])
        print(self.sigma)
        for i, line in enumerate(self.VT):
            print('=> Ligne %d' % (i + 1), '(ma note : %f)' % self.U[the_i][i])
            sorted_line = sorted((line[j], self.work_titles[work_ids[j]]) for j in range(nb_works))[::-1]
            top5 = sorted_line[:10]
            bottom5 = sorted_line[-10:]
            for anime in top5:
                print(anime)
            for anime in bottom5:
                print(anime)
            """if i == 0 or i == 1:  # First two vectors explaining variance
                with open('vector%d.json' % (i + 1), 'w') as f:
                    vi = X.dot(line).tolist()
                    x_norm = [np.dot(X.data[k], X.data[k]) / (nb_works + 1) for k in range(nb_users + 1)]
                    f.write(json.dumps({'v': [v / math.sqrt(x_norm[k]) if x_norm[k] != 0 else float('inf') for k, v in enumerate(vi)]}))"""
        # print(VT.dot(VT.transpose()))
        # return

        the_ratings = self.predict((the_user_id, work_ids[j]) for j in range(nb_works))
        ranking = sorted(zip(the_ratings, [(work_ids[j], self.work_titles[work_ids[j]]) for j in range(nb_works)]), reverse=True)

        # Summarize the results of the ranking for the_user_id:
        # “=> rank, title, score”
        c = 0
        for i, (rating, (work_id, title)) in enumerate(ranking, start=1):
            if work_id not in seen_works:
                print('=>', i, title, rating, self.predict([(the_user_id, work_id)]))
                if Recommendation.objects.filter(user=svd_user, target_user__id=the_user_id, work__id=work_id).count() == 0:
                    Recommendation.objects.create(user=svd_user, target_user_id=the_user_id, work_id=work_id)
                c += 1
            elif i < TOP:
                print(i, title, rating)
            if c >= TOP:
                break

        """print(len(connection.queries), 'queries')
        for line in connection.queries:
            print(line)"""

        self.chrono.save('complete')

    def __str__(self):
        return '[SVD]'

    def get_shortname(self):
        return 'svd'


# -*- coding: utf-8 -*-
# Copyright 2007-2011 The HyperSpy developers
#
# This file is part of  HyperSpy.
#
#  HyperSpy is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
#  HyperSpy is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with  HyperSpy.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division

import numpy as np
import matplotlib.pyplot as plt

from hyperspy.drawing.figure import BlittedFigure
from hyperspy.drawing import utils

class HistogramTilePlot(BlittedFigure):

    def __init__(self):
        self.figure = None
        self.title = ''
        self.ax = None

    def create_axis(self, ncols=1, nrows=1, number=1, title=''):
        ax = self.figure.add_subplot(ncols, nrows, number)
        ax.set_title(title)
        ax.hspy_fig = self
        return ax

    def plot(self, db, **kwargs):
        if self.figure is None:
            self.create_figure()
        ncomps = len(db)

        if not ncomps:
            return
        else:
            self.update(db, **kwargs)

    def update(self, db, **kwargs):
        ncomps = len(db)
        # get / set axes
        i = -1
        for c_n, v in db.items():
            i += 1
            ncols = len(v)
            istart = ncols * i
            j = 0
            for p_n, (hist, bin_edges) in v.items():
                j += 1
                mask = hist > 0
                if np.any(mask):
                    title = c_n + ' ' + p_n
                    ax = self.create_axis(ncomps, ncols, istart + j, title)
                    self.ax = ax
                    # remove previous
                    while ax.patches:
                        ax.patches[0].remove()
                    # set new; only draw non-zero height bars
                    ax.bar(
                        bin_edges[
                            :-1][mask],
                        hist[mask],
                        np.diff(bin_edges)[mask],
                        # animated=True,
                        **kwargs)
                    width = bin_edges[-1] - bin_edges[0]
                    ax.set_xlim(
                        bin_edges[0] - width * 0.1, bin_edges[-1] + width * 0.1)
                    ax.set_ylim(0, np.max(hist) * 1.1)
                    # ax.set_title(c_n + ' ' + p_n)
        self.figure.canvas.draw_idle()

    def close(self):
        try:
            plt.close(self.figure)
        except BaseException:
            pass
        self.figure = None


import numpy as np
from sklearn.cluster import KMeans
from sklearn import preprocessing
import pandas as pd

'''
Pclass Passenger Class (1 = 1st; 2 = 2nd; 3 = 3rd)
survival Survival (0 = No; 1 = Yes)
name Name
sex Sex
age Age
sibsp Number of Siblings/Spouses Aboard
parch Number of Parents/Children Aboard
ticket Ticket Number
fare Passenger Fare (British pound)
cabin Cabin
embarked Port of Embarkation (C = Cherbourg; Q = Queenstown; S = Southampton)
boat Lifeboat
body Body Identification Number
home.dest Home/Destination
'''

df = pd.read_excel('titanic.xls')
df.drop(['body', 'name'], 1, inplace=True)
df.fillna(0, inplace=True)

def handle_non_numerical_data(df):
    columns = df.columns.values
    for column in columns:
        text_digit_vals = {}

        def convert_to_int(val):
            return text_digit_vals[val]

        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x += 1
            df[column] = list(map(convert_to_int, df[column]))
    return df

df = handle_non_numerical_data(df)
X = np.array(df.drop(['survived'], 1).astype(float))
X = preprocessing.scale(X)
y = np.array(df['survived'])

clf = KMeans(n_clusters=2)
clf.fit(X)

correct = 0
for i in range(len(X)):
    predict_me = np.array(X[i].astype(float))
    predict_me = predict_me.reshape(-1, len(predict_me))
    prediction = clf.predict(predict_me)
    if prediction[0] == y[i]:
        correct += 1

print(correct/len(X))


import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from scipy.ndimage.filters import maximum_filter
from scipy.ndimage.morphology import (generate_binary_structure,
                                      iterate_structure, binary_erosion)
import hashlib
from operator import itemgetter

IDX_FREQ_I = 0
IDX_TIME_J = 1

######################################################################
# Sampling rate, related to the Nyquist conditions, which affects
# the range frequencies we can detect.
DEFAULT_FS = 44100

######################################################################
# Size of the FFT window, affects frequency granularity
DEFAULT_WINDOW_SIZE = 4096

######################################################################
# Ratio by which each sequential window overlaps the last and the
# next window. Higher overlap will allow a higher granularity of offset
# matching, but potentially more fingerprints.
DEFAULT_OVERLAP_RATIO = 0.5

######################################################################
# Degree to which a fingerprint can be paired with its neighbors --
# higher will cause more fingerprints, but potentially better accuracy.
DEFAULT_FAN_VALUE = 15

######################################################################
# Minimum amplitude in spectrogram in order to be considered a peak.
# This can be raised to reduce number of fingerprints, but can negatively
# affect accuracy.
DEFAULT_AMP_MIN = 10

######################################################################
# Number of cells around an amplitude peak in the spectrogram in order
# for Dejavu to consider it a spectral peak. Higher values mean less
# fingerprints and faster matching, but can potentially affect accuracy.
PEAK_NEIGHBORHOOD_SIZE = 20

######################################################################
# Thresholds on how close or far fingerprints can be in time in order
# to be paired as a fingerprint. If your max is too low, higher values of
# DEFAULT_FAN_VALUE may not perform as expected.
MIN_HASH_TIME_DELTA = 0
MAX_HASH_TIME_DELTA = 200

######################################################################
# If True, will sort peaks temporally for fingerprinting;
# not sorting will cut down number of fingerprints, but potentially
# affect performance.
PEAK_SORT = True

######################################################################
# Number of bits to throw away from the front of the SHA1 hash in the
# fingerprint calculation. The more you throw away, the less storage, but
# potentially higher collisions and misclassifications when identifying songs.
FINGERPRINT_REDUCTION = 20

def fingerprint(channel_samples, Fs=DEFAULT_FS,
                wsize=DEFAULT_WINDOW_SIZE,
                wratio=DEFAULT_OVERLAP_RATIO,
                fan_value=DEFAULT_FAN_VALUE,
                amp_min=DEFAULT_AMP_MIN):
    """
    FFT the channel, log transform output, find local maxima, then return
    locally sensitive hashes.
    """
    # FFT the signal and extract frequency components
    arr2D = mlab.specgram(
        channel_samples,
        NFFT=wsize,
        Fs=Fs,
        window=mlab.window_hanning,
        noverlap=int(wsize * wratio))[0]

    # apply log transform since specgram() returns linear array
    arr2D = 10 * np.log10(arr2D)
    arr2D[arr2D == -np.inf] = 0  # replace infs with zeros

    # find local maxima
    local_maxima = get_2D_peaks(arr2D, plot=False, amp_min=amp_min)

    # return hashes
    return generate_hashes(local_maxima, fan_value=fan_value)

def get_2D_peaks(arr2D, plot=False, amp_min=DEFAULT_AMP_MIN):
    # http://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.morphology.iterate_structure.html#scipy.ndimage.morphology.iterate_structure
    struct = generate_binary_structure(2, 1)
    neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE)

    # find local maxima using our fliter shape
    local_max = maximum_filter(arr2D, footprint=neighborhood) == arr2D
    background = (arr2D == 0)
    eroded_background = binary_erosion(background, structure=neighborhood,
                                       border_value=1)

    # Boolean mask of arr2D with True at peaks
    detected_peaks = local_max - eroded_background

    # extract peaks
    amps = arr2D[detected_peaks]
    j, i = np.where(detected_peaks)

    # filter peaks
    amps = amps.flatten()
    peaks = zip(i, j, amps)
    peaks_filtered = [x for x in peaks if x[2] > amp_min]  # freq, time, amp

    # get indices for frequency and time
    frequency_idx = [x[1] for x in peaks_filtered]
    time_idx = [x[0] for x in peaks_filtered]

    if plot:
        # scatter of the peaks
        fig, ax = plt.subplots()
        ax.imshow(arr2D)
        ax.scatter(time_idx, frequency_idx)
        ax.set_xlabel('Time')
        ax.set_ylabel('Frequency')
        ax.set_title("Spectrogram")
        plt.gca().invert_yaxis()
        plt.show()

    return zip(frequency_idx, time_idx)

def generate_hashes(peaks, fan_value=DEFAULT_FAN_VALUE):
    """
    Hash list structure:
       sha1_hash[0:20]    time_offset
    [(e05b341a9b77a51fd26, 32), ... ]
    """
    fingerprinted = set()  # to avoid rehashing same pairs

    if PEAK_SORT:
        peaks.sort(key=itemgetter(1))

    for i in range(len(peaks)):
        for j in range(1, fan_value):
            if (i + j) < len(peaks) and not (i, i + j) in fingerprinted:
                freq1 = peaks[i][IDX_FREQ_I]
                freq2 = peaks[i + j][IDX_FREQ_I]

                t1 = peaks[i][IDX_TIME_J]
                t2 = peaks[i + j][IDX_TIME_J]

                t_delta = t2 - t1

                if t_delta >= MIN_HASH_TIME_DELTA and t_delta <= MAX_HASH_TIME_DELTA:
                    h = hashlib.sha1(
                        "%s|%s|%s" % (str(freq1), str(freq2), str(t_delta)))
                    yield (h.hexdigest()[0:FINGERPRINT_REDUCTION], t1)

                # ensure we don't repeat hashing
                fingerprinted.add((i, i + j))


# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Example code for TensorFlow Wide & Deep Tutorial using TF.Learn API."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import shutil
import sys
import tempfile

import pandas as pd
from six.moves import urllib
import tensorflow as tf

CSV_COLUMNS = [
    "age", "workclass", "fnlwgt", "education", "education_num",
    "marital_status", "occupation", "relationship", "race", "gender",
    "capital_gain", "capital_loss", "hours_per_week", "native_country",
    "income_bracket"
]

gender = tf.feature_column.categorical_column_with_vocabulary_list(
    "gender", ["Female", "Male"])
education = tf.feature_column.categorical_column_with_vocabulary_list(
    "education", [
        "Bachelors", "HS-grad", "11th", "Masters", "9th",
        "Some-college", "Assoc-acdm", "Assoc-voc", "7th-8th",
        "Doctorate", "Prof-school", "5th-6th", "10th", "1st-4th",
        "Preschool", "12th"
    ])
marital_status = tf.feature_column.categorical_column_with_vocabulary_list(
    "marital_status", [
        "Married-civ-spouse", "Divorced", "Married-spouse-absent",
        "Never-married", "Separated", "Married-AF-spouse", "Widowed"
    ])
relationship = tf.feature_column.categorical_column_with_vocabulary_list(
    "relationship", [
        "Husband", "Not-in-family", "Wife", "Own-child", "Unmarried",
        "Other-relative"
    ])
workclass = tf.feature_column.categorical_column_with_vocabulary_list(
    "workclass", [
        "Self-emp-not-inc", "Private", "State-gov", "Federal-gov",
        "Local-gov", "?", "Self-emp-inc", "Without-pay", "Never-worked"
    ])

# To show an example of hashing:
occupation = tf.feature_column.categorical_column_with_hash_bucket(
    "occupation", hash_bucket_size=1000)
native_country = tf.feature_column.categorical_column_with_hash_bucket(
    "native_country", hash_bucket_size=1000)

# Continuous base columns.
age = tf.feature_column.numeric_column("age")
education_num = tf.feature_column.numeric_column("education_num")
capital_gain = tf.feature_column.numeric_column("capital_gain")
capital_loss = tf.feature_column.numeric_column("capital_loss")
hours_per_week = tf.feature_column.numeric_column("hours_per_week")

# Transformations.
age_buckets = tf.feature_column.bucketized_column(
    age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])

# Wide columns and deep columns.
base_columns = [
    gender, education, marital_status, relationship, workclass, occupation,
    native_country, age_buckets,
]

crossed_columns = [
    tf.feature_column.crossed_column(
        ["education", "occupation"], hash_bucket_size=1000),
    tf.feature_column.crossed_column(
        [age_buckets, "education", "occupation"], hash_bucket_size=1000),
    tf.feature_column.crossed_column(
        ["native_country", "occupation"], hash_bucket_size=1000)
]

deep_columns = [
    tf.feature_column.indicator_column(workclass),
    tf.feature_column.indicator_column(education),
    tf.feature_column.indicator_column(gender),
    tf.feature_column.indicator_column(relationship),
    # To show an example of embedding
    tf.feature_column.embedding_column(native_country, dimension=8),
    tf.feature_column.embedding_column(occupation, dimension=8),
    age,
    education_num,
    capital_gain,
    capital_loss,
    hours_per_week,
]

def maybe_download(train_data, test_data):
  """Maybe downloads training data and returns train and test file names."""
  if train_data:
    train_file_name = train_data
  else:
    train_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data",
        train_file.name)  # pylint: disable=line-too-long
    train_file_name = train_file.name
    train_file.close()
    print("Training data is downloaded to %s" % train_file_name)

  if test_data:
    test_file_name = test_data
  else:
    test_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test",
        test_file.name)  # pylint: disable=line-too-long
    test_file_name = test_file.name
    test_file.close()
    print("Test data is downloaded to %s"% test_file_name)

  return train_file_name, test_file_name

def build_estimator(model_dir, model_type):
  """Build an estimator."""
  if model_type == "wide":
    m = tf.estimator.LinearClassifier(
        model_dir=model_dir, feature_columns=base_columns + crossed_columns)
  elif model_type == "deep":
    m = tf.estimator.DNNClassifier(
        model_dir=model_dir,
        feature_columns=deep_columns,
        hidden_units=[100, 50])
  else:
    m = tf.estimator.DNNLinearCombinedClassifier(
        model_dir=model_dir,
        linear_feature_columns=crossed_columns,
        dnn_feature_columns=deep_columns,
        dnn_hidden_units=[100, 50])
  return m

def input_fn(data_file, num_epochs, shuffle):
  """Input builder function."""
  df_data = pd.read_csv(
      tf.gfile.Open(data_file),
      names=CSV_COLUMNS,
      skipinitialspace=True,
      engine="python",
      skiprows=1)
  # remove NaN elements
  df_data = df_data.dropna(how="any", axis=0)
  labels = df_data["income_bracket"].apply(lambda x: ">50K" in x).astype(int)
  return tf.estimator.inputs.pandas_input_fn(
      x=df_data,
      y=labels,
      batch_size=100,
      num_epochs=num_epochs,
      shuffle=shuffle,
      num_threads=5)

def train_and_eval(model_dir, model_type, train_steps, train_data, test_data):
  """Train and evaluate the model."""
  train_file_name, test_file_name = maybe_download(train_data, test_data)
  # Specify file path below if want to find the output easily
  model_dir = tempfile.mkdtemp() if not model_dir else model_dir

  m = build_estimator(model_dir, model_type)
  # set num_epochs to None to get infinite stream of data.
  m.train(
      input_fn=input_fn(train_file_name, num_epochs=None, shuffle=True),
      steps=train_steps)
  # set steps to None to run evaluation until all data consumed.
  results = m.evaluate(
      input_fn=input_fn(test_file_name, num_epochs=1, shuffle=False),
      steps=None)
  print("model directory = %s" % model_dir)
  for key in sorted(results):
    print("%s: %s" % (key, results[key]))
  # Manual cleanup
  shutil.rmtree(model_dir)

FLAGS = None

def main(_):
  train_and_eval(FLAGS.model_dir, FLAGS.model_type, FLAGS.train_steps,
                 FLAGS.train_data, FLAGS.test_data)

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.register("type", "bool", lambda v: v.lower() == "true")
  parser.add_argument(
      "--model_dir",
      type=str,
      default="",
      help="Base directory for output models."
  )
  parser.add_argument(
      "--model_type",
      type=str,
      default="wide_n_deep",
      help="Valid model types: {'wide', 'deep', 'wide_n_deep'}."
  )
  parser.add_argument(
      "--train_steps",
      type=int,
      default=2000,
      help="Number of training steps."
  )
  parser.add_argument(
      "--train_data",
      type=str,
      default="",
      help="Path to the training data."
  )
  parser.add_argument(
      "--test_data",
      type=str,
      default="",
      help="Path to the test data."
  )
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


"""
Testing for Clustering methods

"""

import numpy as np

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises

from sklearn.cluster.affinity_propagation_ import AffinityPropagation
from sklearn.cluster.affinity_propagation_ import affinity_propagation
from sklearn.datasets.samples_generator import make_blobs
from sklearn.metrics import euclidean_distances

n_clusters = 3
centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10
X, _ = make_blobs(n_samples=60, n_features=2, centers=centers,
                  cluster_std=0.4, shuffle=True, random_state=0)

def test_affinity_propagation():
    # Affinity Propagation algorithm
    # Compute similarities
    S = -euclidean_distances(X, squared=True)
    preference = np.median(S) * 10
    # Compute Affinity Propagation
    cluster_centers_indices, labels = affinity_propagation(
        S, preference=preference)

    n_clusters_ = len(cluster_centers_indices)

    assert_equal(n_clusters, n_clusters_)

    af = AffinityPropagation(preference=preference, affinity="precomputed")
    labels_precomputed = af.fit(S).labels_

    af = AffinityPropagation(preference=preference, verbose=True)
    labels = af.fit(X).labels_

    assert_array_equal(labels, labels_precomputed)

    cluster_centers_indices = af.cluster_centers_indices_

    n_clusters_ = len(cluster_centers_indices)
    assert_equal(np.unique(labels).size, n_clusters_)
    assert_equal(n_clusters, n_clusters_)

    # Test also with no copy
    _, labels_no_copy = affinity_propagation(S, preference=preference,
                                             copy=False)
    assert_array_equal(labels, labels_no_copy)

    # Test input validation
    assert_raises(ValueError, affinity_propagation, S[:, :-1])
    assert_raises(ValueError, affinity_propagation, S, damping=0)
    af = AffinityPropagation(affinity="unknown")
    assert_raises(ValueError, af.fit, X)

def test_affinity_propagation_predict():
    # Test AffinityPropagation.predict
    af = AffinityPropagation(affinity="euclidean")
    labels = af.fit_predict(X)
    labels2 = af.predict(X)
    assert_array_equal(labels, labels2)

def test_affinity_propagation_predict_error():
    # Test exception in AffinityPropagation.predict
    # Not fitted.
    af = AffinityPropagation(affinity="euclidean")
    assert_raises(ValueError, af.predict, X)

    # Predict not supported when affinity="precomputed".
    S = np.dot(X, X.T)
    af = AffinityPropagation(affinity="precomputed")
    af.fit(S)
    assert_raises(ValueError, af.predict, X)


import nltk
import pandas as pd
import re
from googletrans import Translator
from unicodedata import normalize

def read_csv(file):

    df1 = pd.DataFrame.from_csv('files_extern/%s.csv'%(file),sep=';',index_col=0,encoding ='ISO-8859-1')

    df1 = df1.reset_index()

    return df1

def write_csv(data,file):
    df = pd.DataFrame(data)
    df.to_csv('files_extern/'+file+'.csv', mode='w', sep=';',index=False, header=False,encoding='utf8')

def clear(dataframe):
    new_df_tweet = []
    new_df_sent = []
    zipped = zip(dataframe['tweet'],dataframe['opiniao'])
    for (df,opiniao) in zipped:
        expr = re.sub(r"http\S+", "", df)
        #expr = re.sub(r"[@#]\S+","",expr)
        expr = normalize('NFKD',expr).encode('ASCII','ignore').decode('ASCII')
        filtrado = [w for w in nltk.regexp_tokenize(expr.lower(),"[^0-9\W_]+") if not w in nltk.corpus.stopwords.words('portuguese')]
        for f in filtrado:
            if len(f) >= 2:
                #print(f)
                #print(opiniao)
                new_df_tweet.append(f)
                new_df_sent.append(opiniao)

    new_df = pd.DataFrame()

    new_df['tokens'] = new_df_tweet
    new_df['sentimento'] = new_df_sent

    return new_df

def convert_df(df):
    new_df = []
    for d in df:
        if d == 'Positivo':
            new_df.append(1)
        elif d == 'Neutro':
            new_df.append(0)
        elif d == 'Negativo':
            new_df.append(-1)

    return new_df

def exlusivos(vet_neg,vet_neu,vet_pos):
	ex_pos = []
	ex_neg = []
	ex_neu = []
	tupla = zip(vet_neg,vet_neu,vet_pos)
	for (neg,neu,pos) in tupla:
		if  not (neg in vet_pos or neg in vet_neu):
			ex_neg.append(neg)
		if  not (neu in vet_neg or neu in vet_pos):
			ex_neu.append(neu)
		if not (pos in vet_neg or pos in vet_neu):
			ex_pos.append(pos)

	print(ex_neg)
	print(ex_neu)
	print(ex_pos)

	return ex_neg, ex_neu, ex_pos

def bigram(frases,vet_neg, vet_neu,vet_pos):
    bi_neg = []
    bi_neu = []
    bi_pos = []
    for f in frases:
        if f.find()

if __name__ == '__main__':

    df_tweets = read_csv('dataset-portuguese')
    df_tweets['opiniao'] = convert_df(df_tweets['opiniao'])
    df_words = clear(df_tweets)

    neg = df_words.loc[df_words['sentimento'] == -1]
    neu = df_words.loc[df_words['sentimento'] == 0]
    pos = df_words.loc[df_words['sentimento'] == 1]

    neg_freq = nltk.FreqDist(neg['tokens'])
    neu_freq = nltk.FreqDist(neu['tokens'])
    pos_freq = nltk.FreqDist(pos['tokens'])

    vet_neg = []
    vet_neu = []
    vet_pos = []

    #neg_freq.plot(50, cumulative=False)
    #neu_freq.plot(50, cumulative=False)
    #pos_freq.plot(50, cumulative=False)

    #print(neg_freq.most_common(30))
    #print('------------------------')
    #print(neu_freq.most_common(30))
    #print('------------------------')
    #print(pos_freq.most_common(30))

    tupla = zip(neg_freq.most_common(len(neg)),neu_freq.most_common(len(neu)),pos_freq.most_common(len(pos)))

    df_neg = pd.DataFrame()
    df_neu = pd.DataFrame()
    df_pos = pd.DataFrame()

    words_neg = dict()
    words_neu = dict()
    words_pos = dict()

    words_neg['pt'] = []
    words_neg['en'] = []
    words_neg['es'] = []
    words_neu['pt'] = []
    words_neu['en'] = []
    words_neu['es'] = []
    words_pos['pt'] = []
    words_pos['en'] = []
    words_pos['es'] = []

    #neg_freq.plot(30, cumulative=False)

    translator = Translator(service_urls=['translate.google.com','translate.google.com.br'])

    for (ng,nu,ps) in tupla:
    	vet_neg.append(ng[0])
    	vet_neu.append(nu[0])
    	vet_pos.append(ps[0])

    vet_neg, vet_neu,vet_pos = exlusivos(vet_neg,vet_neu,vet_pos)

    tupla = zip(vet_neg[:50],vet_neu[:50],vet_pos[:50])

    for (ng,nu,ps) in tupla:
    	words_neg['pt'].append(ng)
    	en=translator.translate(ng, dest='en').text
    	words_neg['en'].append(en)
    	words_neg['es'].append(translator.translate(en, dest='es').text)

    	words_neu['pt'].append(nu)
    	en=translator.translate(nu, dest='en').text
    	words_neu['en'].append(en)
    	words_neu['es'].append(translator.translate(en, dest='es').text)

    	words_pos['pt'].append(ps)
    	en=translator.translate(ps, dest='en').text
    	words_pos['en'].append(en)
    	words_pos['es'].append(translator.translate(en, dest='es').text)

    df_neg['pt'] = words_neg['pt']
    df_neg['en'] = words_neg['en']
    df_neg['es'] = words_neg['es']

    df_neu['pt'] = words_neu['pt']
    df_neu['en'] = words_neu['en']
    df_neu['es'] = words_neu['es']

    df_pos['pt'] = words_pos['pt']
    df_pos['en'] = words_pos['en']
    df_pos['es'] = words_pos['es']

    write_csv(df_neg,'bigram_neg')
    write_csv(df_neu,'bigram_neu')
    write_csv(df_pos,'bigram_pos')


"""
============================
Faces dataset decompositions
============================

This example applies to :ref:`olivetti_faces` different unsupervised
matrix decomposition (dimension reduction) methods from the module
:py:mod:`sklearn.decomposition` (see the documentation chapter
:ref:`decompositions`) .

"""
print(__doc__)

# Authors: Vlad Niculae, Alexandre Gramfort
# License: BSD 3 clause

import logging
from time import time

from numpy.random import RandomState
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_olivetti_faces
from sklearn.cluster import MiniBatchKMeans
from sklearn import decomposition

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')
n_row, n_col = 2, 3
n_components = n_row * n_col
image_shape = (64, 64)
rng = RandomState(0)

###############################################################################
# Load faces data
dataset = fetch_olivetti_faces(shuffle=True, random_state=rng)
faces = dataset.data

n_samples, n_features = faces.shape

# global centering
faces_centered = faces - faces.mean(axis=0)

# local centering
faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)

print("Dataset consists of %d faces" % n_samples)

###############################################################################
def plot_gallery(title, images, n_col=n_col, n_row=n_row):
    plt.figure(figsize=(2. * n_col, 2.26 * n_row))
    plt.suptitle(title, size=16)
    for i, comp in enumerate(images):
        plt.subplot(n_row, n_col, i + 1)
        vmax = max(comp.max(), -comp.min())
        plt.imshow(comp.reshape(image_shape), cmap=plt.cm.gray,
                   interpolation='nearest',
                   vmin=-vmax, vmax=vmax)
        plt.xticks(())
        plt.yticks(())
    plt.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.)

###############################################################################
# List of the different estimators, whether to center and transpose the
# problem, and whether the transformer uses the clustering API.
estimators = [
    ('Eigenfaces - RandomizedPCA',
     decomposition.RandomizedPCA(n_components=n_components, whiten=True),
     True),

    ('Non-negative components - NMF',
     decomposition.NMF(n_components=n_components, init='nndsvda', beta=5.0,
                       tol=5e-3, sparseness='components'),
     False),

    ('Independent components - FastICA',
     decomposition.FastICA(n_components=n_components, whiten=True),
     True),

    ('Sparse comp. - MiniBatchSparsePCA',
     decomposition.MiniBatchSparsePCA(n_components=n_components, alpha=0.8,
                                      n_iter=100, batch_size=3,
                                      random_state=rng),
     True),

    ('MiniBatchDictionaryLearning',
        decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                  n_iter=50, batch_size=3,
                                                  random_state=rng),
     True),

    ('Cluster centers - MiniBatchKMeans',
        MiniBatchKMeans(n_clusters=n_components, tol=1e-3, batch_size=20,
                        max_iter=50, random_state=rng),
     True),

    ('Factor Analysis components - FA',
     decomposition.FactorAnalysis(n_components=n_components, max_iter=2),
     True),
]

###############################################################################
# Plot a sample of the input data

plot_gallery("First centered Olivetti faces", faces_centered[:n_components])

###############################################################################
# Do the estimation and plot it

for name, estimator, center in estimators:
    print("Extracting the top %d %s..." % (n_components, name))
    t0 = time()
    data = faces
    if center:
        data = faces_centered
    estimator.fit(data)
    train_time = (time() - t0)
    print("done in %0.3fs" % train_time)
    if hasattr(estimator, 'cluster_centers_'):
        components_ = estimator.cluster_centers_
    else:
        components_ = estimator.components_
    if hasattr(estimator, 'noise_variance_'):
        plot_gallery("Pixelwise variance",
                     estimator.noise_variance_.reshape(1, -1), n_col=1,
                     n_row=1)
    plot_gallery('%s - Train time %.1fs' % (name, train_time),
                 components_[:n_components])

plt.show()


"""
Problem 3.

calculate the time series

yt = 5 + .05 * t + Et (Where E is epsilon)

for years 1960, 1961, ..., 2001 assuming Et independently and
identically distributed with mean 0 and sigma 0.2.
"""
from random import uniform

from matplotlib.pyplot import plot, show
from numpy import array, polyfit, poly1d

def create_distribution(size):
    """
    Create a distribution, identically distributed, with mean 0 and
    sigma 0.2
    """
    # Shit it's way easier to just do some uniform distribution
    # This is a bit over my head, and not possible for me without
    # pen and paper
    return array([uniform(-0.2, .2) for _ in xrange(size)])

def create_time_series(start_year, end_year):
    """
    Create the time series, yt, then perform a regress on yt, plot yt and the
    its trendline
    """
    t_array = array(range(start_year, end_year + 1))
    epsilon_t = create_distribution(len(t_array))
    yt = array([5 + .05 * t_i + epsilon_t[i] for i, t_i in enumerate(t_array)])
    fit = polyfit(t_array, yt, 1)
    fit_func = poly1d(fit)
    plot(t_array, yt, "yo", t_array, fit_func(t_array), "--k")
    show()

def main():
    create_time_series(1960, 2001)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
#
# Copyright 2009 Free Software Foundation, Inc.
#
# This file is part of GNU Radio
#
# GNU Radio is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GNU Radio is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNU Radio; see the file COPYING.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street,
# Boston, MA 02110-1301, USA.
#

from gnuradio import gr, blks2
import sys, time

try:
    import scipy
    from scipy import fftpack
except ImportError:
    print "Error: Program requires scipy (see: www.scipy.org)."
    sys.exit(1)

try:
    import pylab
    from pylab import mlab
except ImportError:
    print "Error: Program requires matplotlib (see: matplotlib.sourceforge.net)."
    sys.exit(1)

class pfb_top_block(gr.top_block):
    def __init__(self):
        gr.top_block.__init__(self)

        self._N = 100000        # number of samples to use
        self._fs = 2000         # initial sampling rate
        self._interp = 5        # Interpolation rate for PFB interpolator
        self._ainterp = 5.5       # Resampling rate for the PFB arbitrary resampler

        # Frequencies of the signals we construct
        freq1 = 100
        freq2 = 200

        # Create a set of taps for the PFB interpolator
        # This is based on the post-interpolation sample rate
        self._taps = gr.firdes.low_pass_2(self._interp, self._interp*self._fs, freq2+50, 50,
                                          attenuation_dB=120, window=gr.firdes.WIN_BLACKMAN_hARRIS)

        # Create a set of taps for the PFB arbitrary resampler
        # The filter size is the number of filters in the filterbank; 32 will give very low side-lobes,
        # and larger numbers will reduce these even farther
        # The taps in this filter are based on a sampling rate of the filter size since it acts
        # internally as an interpolator.
        flt_size = 32
        self._taps2 = gr.firdes.low_pass_2(flt_size, flt_size*self._fs, freq2+50, 150,
                                           attenuation_dB=120, window=gr.firdes.WIN_BLACKMAN_hARRIS)

        # Calculate the number of taps per channel for our own information
        tpc = scipy.ceil(float(len(self._taps)) /  float(self._interp))
        print "Number of taps:     ", len(self._taps)
        print "Number of filters:  ", self._interp
        print "Taps per channel:   ", tpc

        # Create a couple of signals at different frequencies
        self.signal1 = gr.sig_source_c(self._fs, gr.GR_SIN_WAVE, freq1, 0.5)
        self.signal2 = gr.sig_source_c(self._fs, gr.GR_SIN_WAVE, freq2, 0.5)
        self.signal = gr.add_cc()

        self.head = gr.head(gr.sizeof_gr_complex, self._N)

        # Construct the PFB interpolator filter
        self.pfb = blks2.pfb_interpolator_ccf(self._interp, self._taps)

        # Construct the PFB arbitrary resampler filter
        self.pfb_ar = blks2.pfb_arb_resampler_ccf(self._ainterp, self._taps2, flt_size)
        self.snk_i = gr.vector_sink_c()

        #self.pfb_ar.pfb.print_taps()
        #self.pfb.pfb.print_taps()

        # Connect the blocks
        self.connect(self.signal1, self.head, (self.signal,0))
        self.connect(self.signal2, (self.signal,1))
        self.connect(self.signal, self.pfb)
        self.connect(self.signal, self.pfb_ar)
        self.connect(self.signal, self.snk_i)

        # Create the sink for the interpolated signals
        self.snk1 = gr.vector_sink_c()
        self.snk2 = gr.vector_sink_c()
        self.connect(self.pfb, self.snk1)
        self.connect(self.pfb_ar, self.snk2)

def main():
    tb = pfb_top_block()

    tstart = time.time()
    tb.run()
    tend = time.time()
    print "Run time: %f" % (tend - tstart)

    if 1:
        fig1 = pylab.figure(1, figsize=(12,10), facecolor="w")
        fig2 = pylab.figure(2, figsize=(12,10), facecolor="w")
        fig3 = pylab.figure(3, figsize=(12,10), facecolor="w")

        Ns = 10000
        Ne = 10000

        fftlen = 8192
        winfunc = scipy.blackman

        # Plot input signal
        fs = tb._fs

        d = tb.snk_i.data()[Ns:Ns+Ne]
        sp1_f = fig1.add_subplot(2, 1, 1)

        X,freq = mlab.psd(d, NFFT=fftlen, noverlap=fftlen/4, Fs=fs,
                          window = lambda d: d*winfunc(fftlen),
                          scale_by_freq=True)
        X_in = 10.0*scipy.log10(abs(fftpack.fftshift(X)))
        f_in = scipy.arange(-fs/2.0, fs/2.0, fs/float(X_in.size))
        p1_f = sp1_f.plot(f_in, X_in, "b")
        sp1_f.set_xlim([min(f_in), max(f_in)+1])
        sp1_f.set_ylim([-200.0, 50.0])

        sp1_f.set_title("Input Signal", weight="bold")
        sp1_f.set_xlabel("Frequency (Hz)")
        sp1_f.set_ylabel("Power (dBW)")

        Ts = 1.0/fs
        Tmax = len(d)*Ts

        t_in = scipy.arange(0, Tmax, Ts)
        x_in = scipy.array(d)
        sp1_t = fig1.add_subplot(2, 1, 2)
        p1_t = sp1_t.plot(t_in, x_in.real, "b-o")
        #p1_t = sp1_t.plot(t_in, x_in.imag, "r-o")
        sp1_t.set_ylim([-2.5, 2.5])

        sp1_t.set_title("Input Signal", weight="bold")
        sp1_t.set_xlabel("Time (s)")
        sp1_t.set_ylabel("Amplitude")

        # Plot output of PFB interpolator
        fs_int = tb._fs*tb._interp

        sp2_f = fig2.add_subplot(2, 1, 1)
        d = tb.snk1.data()[Ns:Ns+(tb._interp*Ne)]
        X,freq = mlab.psd(d, NFFT=fftlen, noverlap=fftlen/4, Fs=fs,
                          window = lambda d: d*winfunc(fftlen),
                          scale_by_freq=True)
        X_o = 10.0*scipy.log10(abs(fftpack.fftshift(X)))
        f_o = scipy.arange(-fs_int/2.0, fs_int/2.0, fs_int/float(X_o.size))
        p2_f = sp2_f.plot(f_o, X_o, "b")
        sp2_f.set_xlim([min(f_o), max(f_o)+1])
        sp2_f.set_ylim([-200.0, 50.0])

        sp2_f.set_title("Output Signal from PFB Interpolator", weight="bold")
        sp2_f.set_xlabel("Frequency (Hz)")
        sp2_f.set_ylabel("Power (dBW)")

        Ts_int = 1.0/fs_int
        Tmax = len(d)*Ts_int

        t_o = scipy.arange(0, Tmax, Ts_int)
        x_o1 = scipy.array(d)
        sp2_t = fig2.add_subplot(2, 1, 2)
        p2_t = sp2_t.plot(t_o, x_o1.real, "b-o")
        #p2_t = sp2_t.plot(t_o, x_o.imag, "r-o")
        sp2_t.set_ylim([-2.5, 2.5])

        sp2_t.set_title("Output Signal from PFB Interpolator", weight="bold")
        sp2_t.set_xlabel("Time (s)")
        sp2_t.set_ylabel("Amplitude")

        # Plot output of PFB arbitrary resampler
        fs_aint = tb._fs * tb._ainterp

        sp3_f = fig3.add_subplot(2, 1, 1)
        d = tb.snk2.data()[Ns:Ns+(tb._interp*Ne)]
        X,freq = mlab.psd(d, NFFT=fftlen, noverlap=fftlen/4, Fs=fs,
                          window = lambda d: d*winfunc(fftlen),
                          scale_by_freq=True)
        X_o = 10.0*scipy.log10(abs(fftpack.fftshift(X)))
        f_o = scipy.arange(-fs_aint/2.0, fs_aint/2.0, fs_aint/float(X_o.size))
        p3_f = sp3_f.plot(f_o, X_o, "b")
        sp3_f.set_xlim([min(f_o), max(f_o)+1])
        sp3_f.set_ylim([-200.0, 50.0])

        sp3_f.set_title("Output Signal from PFB Arbitrary Resampler", weight="bold")
        sp3_f.set_xlabel("Frequency (Hz)")
        sp3_f.set_ylabel("Power (dBW)")

        Ts_aint = 1.0/fs_aint
        Tmax = len(d)*Ts_aint

        t_o = scipy.arange(0, Tmax, Ts_aint)
        x_o2 = scipy.array(d)
        sp3_f = fig3.add_subplot(2, 1, 2)
        p3_f = sp3_f.plot(t_o, x_o2.real, "b-o")
        p3_f = sp3_f.plot(t_o, x_o1.real, "m-o")
        #p3_f = sp3_f.plot(t_o, x_o2.imag, "r-o")
        sp3_f.set_ylim([-2.5, 2.5])

        sp3_f.set_title("Output Signal from PFB Arbitrary Resampler", weight="bold")
        sp3_f.set_xlabel("Time (s)")
        sp3_f.set_ylabel("Amplitude")

        pylab.show()

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        pass


"""Test truncated SVD transformer."""

import numpy as np
import scipy.sparse as sp

from sklearn.decomposition import TruncatedSVD
from sklearn.utils import check_random_state
from sklearn.utils.testing import (assert_array_almost_equal, assert_equal,
                                   assert_raises, assert_greater,
                                   assert_array_less)

# Make an X that looks somewhat like a small tf-idf matrix.
# XXX newer versions of SciPy have scipy.sparse.rand for this.
shape = 60, 55
n_samples, n_features = shape
rng = check_random_state(42)
X = rng.randint(-100, 20, np.product(shape)).reshape(shape)
X = sp.csr_matrix(np.maximum(X, 0), dtype=np.float64)
X.data[:] = 1 + np.log(X.data)
Xdense = X.A

def test_algorithms():
    svd_a = TruncatedSVD(30, algorithm="arpack")
    svd_r = TruncatedSVD(30, algorithm="randomized", random_state=42)

    Xa = svd_a.fit_transform(X)[:, :6]
    Xr = svd_r.fit_transform(X)[:, :6]
    assert_array_almost_equal(Xa, Xr, decimal=5)

    comp_a = np.abs(svd_a.components_)
    comp_r = np.abs(svd_r.components_)
    # All elements are equal, but some elements are more equal than others.
    assert_array_almost_equal(comp_a[:9], comp_r[:9])
    assert_array_almost_equal(comp_a[9:], comp_r[9:], decimal=2)

def test_attributes():
    for n_components in (10, 25, 41):
        tsvd = TruncatedSVD(n_components).fit(X)
        assert_equal(tsvd.n_components, n_components)
        assert_equal(tsvd.components_.shape, (n_components, n_features))

def test_too_many_components():
    for algorithm in ["arpack", "randomized"]:
        for n_components in (n_features, n_features + 1):
            tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm)
            assert_raises(ValueError, tsvd.fit, X)

def test_sparse_formats():
    for fmt in ("array", "csr", "csc", "coo", "lil"):
        Xfmt = Xdense if fmt == "dense" else getattr(X, "to" + fmt)()
        tsvd = TruncatedSVD(n_components=11)
        Xtrans = tsvd.fit_transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))
        Xtrans = tsvd.transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))

def test_inverse_transform():
    for algo in ("arpack", "randomized"):
        # We need a lot of components for the reconstruction to be "almost
        # equal" in all positions. XXX Test means or sums instead?
        tsvd = TruncatedSVD(n_components=52, random_state=42)
        Xt = tsvd.fit_transform(X)
        Xinv = tsvd.inverse_transform(Xt)
        assert_array_almost_equal(Xinv, Xdense, decimal=1)

def test_integers():
    Xint = X.astype(np.int64)
    tsvd = TruncatedSVD(n_components=6)
    Xtrans = tsvd.fit_transform(Xint)
    assert_equal(Xtrans.shape, (n_samples, tsvd.n_components))

def test_explained_variance():
    # Test sparse data
    svd_a_10_sp = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_sp = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_sp = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_sp = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_sp = svd_a_10_sp.fit_transform(X)
    X_trans_r_10_sp = svd_r_10_sp.fit_transform(X)
    X_trans_a_20_sp = svd_a_20_sp.fit_transform(X)
    X_trans_r_20_sp = svd_r_20_sp.fit_transform(X)

    # Test dense data
    svd_a_10_de = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_de = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_de = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_de = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_de = svd_a_10_de.fit_transform(X.toarray())
    X_trans_r_10_de = svd_r_10_de.fit_transform(X.toarray())
    X_trans_a_20_de = svd_a_20_de.fit_transform(X.toarray())
    X_trans_r_20_de = svd_r_20_de.fit_transform(X.toarray())

    # helper arrays for tests below
    svds = (svd_a_10_sp, svd_r_10_sp, svd_a_20_sp, svd_r_20_sp, svd_a_10_de,
            svd_r_10_de, svd_a_20_de, svd_r_20_de)
    svds_trans = (
        (svd_a_10_sp, X_trans_a_10_sp),
        (svd_r_10_sp, X_trans_r_10_sp),
        (svd_a_20_sp, X_trans_a_20_sp),
        (svd_r_20_sp, X_trans_r_20_sp),
        (svd_a_10_de, X_trans_a_10_de),
        (svd_r_10_de, X_trans_r_10_de),
        (svd_a_20_de, X_trans_a_20_de),
        (svd_r_20_de, X_trans_r_20_de),
    )
    svds_10_v_20 = (
        (svd_a_10_sp, svd_a_20_sp),
        (svd_r_10_sp, svd_r_20_sp),
        (svd_a_10_de, svd_a_20_de),
        (svd_r_10_de, svd_r_20_de),
    )
    svds_sparse_v_dense = (
        (svd_a_10_sp, svd_a_10_de),
        (svd_a_20_sp, svd_a_20_de),
        (svd_r_10_sp, svd_r_10_de),
        (svd_r_20_sp, svd_r_20_de),
    )

    # Assert the 1st component is equal
    for svd_10, svd_20 in svds_10_v_20:
        assert_array_almost_equal(
            svd_10.explained_variance_ratio_,
            svd_20.explained_variance_ratio_[:10],
            decimal=5,
        )

    # Assert that 20 components has higher explained variance than 10
    for svd_10, svd_20 in svds_10_v_20:
        assert_greater(
            svd_20.explained_variance_ratio_.sum(),
            svd_10.explained_variance_ratio_.sum(),
        )

    # Assert that all the values are greater than 0
    for svd in svds:
        assert_array_less(0.0, svd.explained_variance_ratio_)

    # Assert that total explained variance is less than 1
    for svd in svds:
        assert_array_less(svd.explained_variance_ratio_.sum(), 1.0)

    # Compare sparse vs. dense
    for svd_sparse, svd_dense in svds_sparse_v_dense:
        assert_array_almost_equal(svd_sparse.explained_variance_ratio_,
                                  svd_dense.explained_variance_ratio_)

    # Test that explained_variance is correct
    for svd, transformed in svds_trans:
        total_variance = np.var(X.toarray(), axis=0).sum()
        variances = np.var(transformed, axis=0)
        true_explained_variance_ratio = variances / total_variance

        assert_array_almost_equal(
            svd.explained_variance_ratio_,
            true_explained_variance_ratio,
        )


# encoding: utf-8
'''
2D surface embedded in 3D
-------------------------
'''
from __future__ import absolute_import
import importlib
import os

import numpy as np

from medianshape.simplicial import pointgen3d, mesh, utils
from medianshape.simplicial.meshgen import meshgen2d

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm

from medianshape.viz import plot2d, plot3d
from distmesh.plotting import axes_simpplot3d
from medianshape.simplicial.utils import boundary_points

def func(x, y, sign=1):
    '''
    :math:`\sin\pi x \cos \pi y`.
    '''
    return np.sin(np.pi*x)*np.cos(np.pi*y)

def sample_surf(scale, step=0.2):
    '''
    Returns a tuple X, Y, Z of a surface for an experiment.
    '''
    x = y = np.arange(-4.0, 4.0, step)
    X, Y = np.meshgrid(x, y)
    from matplotlib.mlab import  bivariate_normal
    '''
    Z1 = bivariate_normal(X, Y, 1.0, 1.0, 0.0, 0.0)
    Z2 = bivariate_normal(X, Y, 1.5, 0.5, 1, 1)
    #Z3 = bivariate_normal(X, Y, 1, 1, -2, -2)
    Z = Z2 - Z1
    '''
    # Ups
    ZU1 = bivariate_normal(X,Y, 1.5, 1, 0,-2)
    ZU2 = bivariate_normal(X, Y, 1.5, 1.5, 4, 1)
    ZU3 = bivariate_normal(X, Y, 1, 1, -4, 1)
    #ZU4 = bivariate_normal(X, Y, 1.5, 1.5, -4, -4)
    #ZU5 = bivariate_normal(X, Y, 1, 1, 4, -4)
    ZU4 = bivariate_normal(X, Y, 4, 0.5, 0, -4)
    # Downs
    ZD1 = bivariate_normal(X, Y, 1.5, 1, 0, 1)
    ZD2 = bivariate_normal(X, Y, 1.5, 1.5, -4, -2)
    ZD3 = bivariate_normal(X, Y, 1, 1, 4, -2)
    ZD4 = bivariate_normal(X, Y, 4, 1, 0, 4)
    Z1 = ZU1 + ZU2 + ZU3 - ZD1 - ZD2 - ZD3 - ZD4
    Zmax1 = np.abs(np.amax(Z1))
    Z1 = Z1/Zmax1 * scale[2]

    # Visualization
    fig = plt.figure()
    ax = fig.gca(projection="3d")
    surf = ax.plot_surface(X, Y, Z1, rstride=1, cstride=1, cmap=cm.winter,
                           linewidth=0, antialiased=False)
    plt.show()

    # Ups
    ZU1 = bivariate_normal(X,Y, 2, 1, 1,1)
    ZU2 = bivariate_normal(X, Y, 3, 1, -2, 4)
    ZU3 = bivariate_normal(X, Y, 1.5, 1.5, -2, -2)
    #ZU4 = bivariate_normal(X, Y, 1.5, 1.5, -4, -4)
    #ZU5 = bivariate_normal(X, Y, 1, 1, 4, -4)
    ZU4 = bivariate_normal(X, Y, 2, 2, 3, -4)
    # Downs
    ZD1 = bivariate_normal(X, Y, 1, 2, 4, 2)
    ZD2 = bivariate_normal(X, Y, 1.5, 1.5, -2, 2)
    ZD3 = bivariate_normal(X, Y, 1.5, 1.5, 1, -2)
    ZD4 = bivariate_normal(X, Y, 4, 1, 0, -4)
    Z2 = ZU1 + ZU2 + ZU3 - ZD1 - ZD2 - ZD3 - ZD4
    Zmax2 = np.abs(np.amax(Z2))
    Z2 = Z2/Zmax2 * scale[2]

    X = X * scale[0]/4.0
    Y = Y * scale[1]/4.0

    # Visualization
    fig = plt.figure()
    ax = fig.gca(projection="3d")
    surf = ax.plot_surface(X, Y, Z2, rstride=1, cstride=1, cmap=cm.winter,
                           linewidth=0, antialiased=False)
    plt.show()
    return X, Y, Z1, Z2

def interpolate_surf(points, values, ipoints, method = "nearest"):
    from scipy.interpolate import griddata
    '''
    Used to interpolate a sample surface to a surface in a mesh.
    '''
    return griddata(points, values, ipoints, method= method)

def surfgen_shared_boundary(bbox=[-10,-10,-10, 10,10,10], l=3):
    '''
    Generates two surfaces in 3D with shared boundary for an experiment.
    Writes the two surface as .poly file for tetgen.
    '''
    # Generating point grids for two surfaces
    xmin = bbox[0]
    xmax  = bbox[3]
    ymin = bbox[1]
    ymax = bbox[4]
    zmin = bbox[2]
    zmax = bbox[5]
    Xmin, Ymin, Zmin, Xmax, Ymax, Zmax = np.array(bbox)*0.8
    X, Y, Z1, Z2 = sample_surf([Xmax, Ymax, zmax*0.3], step=0.8)
    Z1 = Z1 + zmax*0.4
    Z2 = Z2 - zmax*0.4
    #Symmertic surfs
    #Z2 = -Z1 - zmax*0.4
    '''
    # Plotting the two surfaces
    fig = plt.figure()
    ax = fig.gca(projection="3d")
    surf = ax.scatter(X, Y, Z1.reshape(-1,1), color='b')
    surf = ax.scatter(X, Y, Z2.reshape(-1,1), color='r')
    plt.show()
    '''
    mesh = meshgen2d([Xmin, Ymin, Xmax, Ymax], l, include_corners=True)
    sample_points = np.hstack((X.reshape(-1,1), Y.reshape(-1,1)))

    # Interpolating the surface mesh into two different surfaces
    # similar to the the sample surfaces generated before
    Z1 = interpolate_surf(sample_points, Z1.reshape(-1,1), mesh.points)
    Z2 = interpolate_surf(sample_points, Z2.reshape(-1,1), mesh.points)

    # Integrating two surfaces
    points1 = np.hstack((mesh.points, Z1))
    print points1.shape
    points2 = np.hstack((mesh.points, Z2))
    print points2.shape
    corners = utils.boundary_points(bbox)
    midcorners = utils.mid_corners(bbox)
    offset1 = len(corners) +len(midcorners) + 1
    offset2 = len(corners) + len(midcorners) + len(points1) + 1
    points = np.concatenate((corners, midcorners, points1, points2), axis=0)
    print points.shape

    triangles1 = mesh.simplices + offset1
    triangles2 = mesh.simplices + offset2

    # Adding the indices of the points as the last column of the coordainate list
    Xmin_s1 = np.argwhere(points1[:,0]==Xmin)
    Xmin_s1_points = np.hstack((points1[Xmin_s1.reshape(-1,)], Xmin_s1))
    # Sorting the indices such that the points are in increasing order of its y-component
    Xmin_s1 = (Xmin_s1_points[:,3][np.argsort(Xmin_s1_points[:,1])] + offset1).astype(int)

    Xmin_s2 = np.argwhere(points2[:,0]==Xmin)
    Xmin_s2_points = np.hstack((points2[Xmin_s2.reshape(-1,)], Xmin_s2))
    Xmin_s2 = (Xmin_s2_points[:,3][np.argsort(Xmin_s2_points[:,1])] + offset2).astype(int)

    Xmax_s1 = np.argwhere(points1[:,0]==Xmax)
    Xmax_s1_points = np.hstack((points1[Xmax_s1.reshape(-1,)], Xmax_s1))
    Xmax_s1 = (Xmax_s1_points[:,3][np.argsort(Xmax_s1_points[:,1])] + offset1).astype(int)

    Xmax_s2 = np.argwhere(points2[:,0]==Xmax)
    Xmax_s2_points = np.hstack((points2[Xmax_s2.reshape(-1,)], Xmax_s2))
    Xmax_s2 = (Xmax_s2_points[:,3][np.argsort(Xmax_s2_points[:,1])] + offset2).astype(int)

    Ymin_s1 = np.argwhere(points1[:,1]==Ymin)
    Ymin_s1_points = np.hstack((points1[Ymin_s1.reshape(-1,)], Ymin_s1))
    Ymin_s1 = (Ymin_s1_points[:,3][np.argsort(Ymin_s1_points[:,0])] + offset1).astype(int)

    Ymin_s2 = np.argwhere(points2[:,1]==Ymin)
    Ymin_s2_points = np.hstack((points2[Ymin_s2.reshape(-1,)], Ymin_s2))
    Ymin_s2 = (Ymin_s2_points[:,3][np.argsort(Ymin_s2_points[:,0])] + offset2).astype(int)

    Ymax_s1 = np.argwhere(points1[:,1]==Ymax)
    Ymax_s1_points = np.hstack((points1[Ymax_s1.reshape(-1,)], Ymax_s1))
    Ymax_s1 = (Ymax_s1_points[:,3][np.argsort(Ymax_s1_points[:,0])] + offset1).astype(int)

    Ymax_s2 = np.argwhere(points2[:,1]==Ymax)
    Ymax_s2_points = np.hstack((points2[Ymax_s2.reshape(-1,)], Ymax_s2))
    Ymax_s2 = (Ymax_s2_points[:,3][np.argsort(Ymax_s2_points[:,0])] + offset2).astype(int)

    for i in range(len(Xmin_s1)-1):
        triangles1 = np.vstack((triangles1, [9, Xmin_s1[i], Xmin_s1[i+1]]))
    triangles1 = np.vstack((triangles1, [9, Xmin_s1[-1], 12]))
    for i in range(len(Xmin_s2)-1):
        triangles2 = np.vstack((triangles2, [9, Xmin_s2[i], Xmin_s2[i+1]]))
    triangles2 = np.vstack((triangles2, [9, Xmin_s2[-1], 12]))
    for i in range(len(Xmax_s1)-1):
        triangles1 = np.vstack((triangles1, [10, Xmax_s1[i], Xmax_s1[i+1]]))
    triangles1 = np.vstack((triangles1, [10, Xmax_s1[-1], 11]))
    for i in range(len(Xmax_s2)-1):
        triangles2 = np.vstack((triangles2, [10, Xmax_s2[i], Xmax_s2[i+1]]))
    triangles2 = np.vstack((triangles2, [10, Xmax_s2[-1], 11]))

    for i in range(len(Ymin_s1)-1):
        triangles1 = np.vstack((triangles1, [9, Ymin_s1[i], Ymin_s1[i+1]]))
    triangles1 = np.vstack((triangles1, [9, Ymin_s1[-1], 10]))
    for i in range(len(Ymin_s2)-1):
        triangles2 = np.vstack((triangles2, [9, Ymin_s2[i], Ymin_s2[i+1]]))
    triangles2 = np.vstack((triangles2, [9, Ymin_s2[-1], 10]))

    for i in range(len(Ymax_s1)-1):
        triangles1 = np.vstack((triangles1, [12, Ymax_s1[i], Ymax_s1[i+1]]))
    triangles1 = np.vstack((triangles1, [12, Ymax_s1[-1], 11]))
    for i in range(len(Ymax_s2)-1):
        triangles2 = np.vstack((triangles2, [12, Ymax_s2[i], Ymax_s2[i+1]]))
    triangles2 = np.vstack((triangles2, [12, Ymax_s2[-1], 11]))

    triangles = np.vstack((triangles1, triangles2))

    # Preparing PLC and save it to .poly file for tetgen
    with open( os.environ['HOME'] +'/mediansurf.poly', 'w') as f:
        f.write("#Part 1 - the node list\n")
        f.write("#%d nodes in 3d, no attributes, no boundary marker\n"%points.shape[0])
        f.write('%d %d %d %d\n'%(points.shape[0], 3, 0,0))
        for i, p in enumerate(points):
            f.write("%d %f %f %f\n"%(i+1, p[0], p[1], p[2]))
        # Each 4 sides has 3 polygons
        # Top and bottom
        # Each triangle of the two surfaces are facets
        fn = 6 + len(triangles)
        f.write("#Part 2 - the facet list.\n")
        f.write("#%d facets with boundary markers\n"%fn)
        f.write('%d %d\n'%(fn, 1))
        f.write("#Boundary facet list.\n")
        f.write("%d %d %d\n"%(1, 0, 1))
        f.write("4 1 2 3 4\n")
        f.write("%d %d %d\n"%(1, 0, 1))
        f.write("4 5 6 7 8\n")
        #xmin side
        f.write("2 0 1\n")
        f.write("4 1 4 8 5\n")
        f.write("2 9 12\n")
        #ymin side
        f.write("2 0 1\n")
        f.write("4 1 2 6 5\n")
        f.write("2 9 10\n")
        #xmax side
        f.write("2 0 1\n")
        f.write("4 2 3 7 6\n")
        f.write("2 10 11\n")
        #ymax side
        f.write("2 0 1\n")
        f.write("4 3 4 8 7\n")
        f.write("2 11 12\n")

        f.write("#Facet list of surface1.\n")
        for t in triangles1:
            f.write("%d %d %d\n"%(1, 0, -1))
            f.write("%d %d %d %d\n"%(3, t[0], t[1], t[2]))
        f.write("#Facet list of surface2.\n")
        for t in triangles2:
            f.write("%d %d %d\n"%(1, 0, -2))
            f.write("%d %d %d %d\n"%(3, t[0], t[1], t[2]))

        f.write("#Part 3 - the hole list.\n")
        f.write('%d\n'%0)
        f.write("#Part 4 - the region list.\n")
        f.write('%d\n'%0)

if __name__ == "__main__":
    surfgen_shared_boundary()


"""
Dragon utilities.
"""

__author__ = "Steven Kearnes"
__copyright__ = "Copyright 2014, Stanford University"
__license__ = "BSD 3-clause"

from cStringIO import StringIO
import numpy as np
import os
import pandas as pd
import subprocess
import tempfile

from vs_utils.utils import SmilesGenerator

class Dragon(object):
    """
    Wrapper for dragon6shell.

    Parameters
    ----------
    subset : str, optional (default '2d')
        Descriptor subset.
    kwargs : dict, optional
        Keyword arguments for SmilesGenerator.
    """
    def __init__(self, subset='2d', **kwargs):
        self.subset = subset
        self.initialized = False
        self.config_filename, self.smiles_engine = None, None
        self.smiles_engine_kwargs = kwargs

    def initialize(self):
        """
        Initialize.

        This is not part of __init__ because it breaks IPython.parallel.
        """
        fd, self.config_filename = tempfile.mkstemp()
        os.close(fd)
        with open(self.config_filename, 'wb') as f:
            f.write(self.get_config())
        self.smiles_engine = SmilesGenerator(**self.smiles_engine_kwargs)
        self.initialized = True

    def __del__(self):
        """
        Cleanup.
        """
        if self.config_filename is not None:
            os.unlink(self.config_filename)

    def get_config(self):
        """
        Get configuration file.
        """
        if self.subset == '2d':
            return """<?xml version="1.0" encoding="utf-8"?>
<DRAGON version="6.0.36" script_version="1" generation_date="2014/11/17">
  <OPTIONS>
    <CheckUpdates value="true"/>
    <SaveLayout value="true"/>
    <ShowWorksheet value="false"/>
    <Decimal_Separator value="."/>
    <Missing_String value="NaN"/>
    <DefaultMolFormat value="1"/>
    <HelpBrowser value="/usr/bin/xdg-open"/>
    <RejectUnusualValence value="false"/>
    <Add2DHydrogens value="false"/>
    <MaxSRforAllCircuit value="19"/>
    <MaxSR value="35"/>
    <MaxSRDetour value="30"/>
    <MaxAtomWalkPath value="2000"/>
    <LogPathWalk value="true"/>
    <LogEdge value="true"/>
    <Weights>
      <weight name="Mass"/>
      <weight name="VdWVolume"/>
      <weight name="Electronegativity"/>
      <weight name="Polarizability"/>
      <weight name="Ionization"/>
      <weight name="I-State"/>
    </Weights>
    <SaveOnlyData value="false"/>
    <SaveLabelsOnSeparateFile value="false"/>
    <SaveFormatBlock value="%b - %n.txt"/>
    <SaveFormatSubBlock value="%b-%s - %n - %m.txt"/>
    <SaveExcludeMisVal value="false"/>
    <SaveExcludeAllMisVal value="false"/>
    <SaveExcludeConst value="false"/>
    <SaveExcludeNearConst value="false"/>
    <SaveExcludeStdDev value="false"/>
    <SaveStdDevThreshold value="0.0001"/>
    <SaveExcludeCorrelated value="false"/>
    <SaveCorrThreshold value="0.95"/>
    <SaveExclusionOptionsToVariables value="false"/>
    <SaveExcludeMisMolecules value="false"/>
    <SaveExcludeRejectedMolecules value="false"/>
  </OPTIONS>
  <DESCRIPTORS>
    <block id="1" SelectAll="true"/>
    <block id="2" SelectAll="true"/>
    <block id="3" SelectAll="true"/>
    <block id="4" SelectAll="true"/>
    <block id="5" SelectAll="true"/>
    <block id="6" SelectAll="true"/>
    <block id="7" SelectAll="true"/>
    <block id="8" SelectAll="true"/>
    <block id="9" SelectAll="true"/>
    <block id="10" SelectAll="true"/>
    <block id="11" SelectAll="true"/>
    <block id="12" SelectAll="true"/>
    <block id="21" SelectAll="true"/>
    <block id="22" SelectAll="true"/>
    <block id="23" SelectAll="true"/>
    <block id="24" SelectAll="true"/>
    <block id="25" SelectAll="true"/>
    <block id="28" SelectAll="true"/>
    <block id="29" SelectAll="true"/>
  </DESCRIPTORS>
  <MOLFILES>
    <molInput value="stdin"/>
    <molInputFormat value="SMILES"/>
  </MOLFILES>
  <OUTPUT>
    <SaveStdOut value="true"/>
    <SaveProject value="false"/>
    <SaveFile value="false"/>
    <logMode value="stderr"/>
  </OUTPUT>
</DRAGON>
"""
        else:
            raise NotImplementedError

    def get_descriptors(self, mols):
        """
        Parameters
        ----------
        mols : array_like
            Molecules.
        """
        if not self.initialized:
            self.initialize()
        smiles = [self.smiles_engine.get_smiles(mol) for mol in mols]
        args = ['dragon6shell', '-s', self.config_filename]
        p = subprocess.Popen(args, stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = p.communicate('\n'.join(smiles))
        if not stdout:
            raise RuntimeError(stderr)
        data, names = self.parse_descriptors(stdout)

        # adjust for skipped molecules
        # descriptors are in same order as smiles
        missing = np.setdiff1d(smiles, names)
        features = np.zeros(len(smiles), dtype=object)
        idx = 0  # index into calculated features
        for i, this_smiles in enumerate(smiles):
            if this_smiles in missing:
                features[i] = None
            else:
                assert this_smiles == names[idx]  # confirm match
                features[i] = data[idx]
                idx += 1
        assert len(features) == len(mols)
        return features

    def parse_descriptors(self, string):
        """
        Parse Dragon descriptors.

        Parameters
        ----------
        string : str
            Output from dragon6shell.
        """
        df = pd.read_table(StringIO(string))
        if self.subset == '2d':
            del df['nHBonds'], df['Psi_e_1d'], df['Psi_e_1s']

        # extract names
        names = df['NAME'].values

        # delete No. and NAME columns
        del df['No.'], df['NAME']

        return np.asarray(df, dtype=float), names


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 23 11:15:12 2016

@author: suraj
"""

import random
import numpy as np
import pickle
import matplotlib.pyplot as plt

attachRateList = []

for i in range(3360):
    attachRateList.append(random.uniform(4,6))

attachRateList = np.array(attachRateList)
encoded_attach_rate_list = np.fft.fft(attachRateList)

day_number_list = [i%7 for i in range(3360)]
encoded_day_number_list = np.fft.fft(day_number_list)

time_number_list = [i%96 for i in range(3360)]
encoded_time_number_list = np.fft.fft(time_number_list)

final_list_x = np.array([[encoded_day_number_list.real[i],encoded_day_number_list.imag[i],encoded_time_number_list.real[i],encoded_time_number_list.imag[i],encoded_attach_rate_list.real[i],encoded_attach_rate_list.imag[i]] for i in range(3360)])

final_list_y = [ (encoded_attach_rate_list[i].real,encoded_attach_rate_list[i].imag) for i in range(len(encoded_attach_rate_list)) ]

pickle.dump(final_list_x,open('x_att.p','wb'))
pickle.dump(final_list_y,open('y_att.p','wb'))