"""
=============================================
Feature Union with Heterogeneous Data Sources
=============================================

Datasets can often contain components of that require different feature
extraction and processing pipelines.  This scenario might occur when:

1. Your dataset consists of heterogeneous data types (e.g. raster images and
   text captions)
2. Your dataset is stored in a Pandas DataFrame and different columns
   require different processing pipelines.

This example demonstrates how to use
:class:`sklearn.feature_extraction.FeatureUnion` on a dataset containing
different types of features.  We use the 20-newsgroups dataset and compute
standard bag-of-words features for the subject line and body in separate
pipelines as well as ad hoc features on the body. We combine them (with
weights) using a FeatureUnion and finally train a classifier on the combined
set of features.

The choice of features is not particularly helpful, but serves to illustrate
the technique.
"""

# Author: Matt Terry <matt.terry@gmail.com>
#
# License: BSD 3 clause
from __future__ import print_function

import numpy as np

from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.datasets import fetch_20newsgroups
from sklearn.datasets.twenty_newsgroups import strip_newsgroup_footer
from sklearn.datasets.twenty_newsgroups import strip_newsgroup_quoting
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn.pipeline import FeatureUnion
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC

class ItemSelector(BaseEstimator, TransformerMixin):
    """For data grouped by feature, select subset of data at a provided key.

    The data is expected to be stored in a 2D data structure, where the first
    index is over features and the second is over samples.  i.e.

    >> len(data[key]) == n_samples

    Please note that this is the opposite convention to scikit-learn feature
    matrixes (where the first index corresponds to sample).

    ItemSelector only requires that the collection implement getitem
    (data[key]).  Examples include: a dict of lists, 2D numpy array, Pandas
    DataFrame, numpy record array, etc.

    >> data = {'a': [1, 5, 2, 5, 2, 8],
               'b': [9, 4, 1, 4, 1, 3]}
    >> ds = ItemSelector(key='a')
    >> data['a'] == ds.transform(data)

    ItemSelector is not designed to handle data grouped by sample.  (e.g. a
    list of dicts).  If your data is structured this way, consider a
    transformer along the lines of `sklearn.feature_extraction.DictVectorizer`.

    Parameters
    ----------
    key : hashable, required
        The key corresponding to the desired value in a mappable.
    """
    def __init__(self, key):
        self.key = key

    def fit(self, x, y=None):
        return self

    def transform(self, data_dict):
        return data_dict[self.key]

class TextStats(BaseEstimator, TransformerMixin):
    """Extract features from each document for DictVectorizer"""

    def fit(self, x, y=None):
        return self

    def transform(self, posts):
        return [{'length': len(text),
                 'num_sentences': text.count('.')}
                for text in posts]

class SubjectBodyExtractor(BaseEstimator, TransformerMixin):
    """Extract the subject & body from a usenet post in a single pass.

    Takes a sequence of strings and produces a dict of sequences.  Keys are
    `subject` and `body`.
    """
    def fit(self, x, y=None):
        return self

    def transform(self, posts):
        features = np.recarray(shape=(len(posts),),
                               dtype=[('subject', object), ('body', object)])
        for i, text in enumerate(posts):
            headers, _, bod = text.partition('\n\n')
            bod = strip_newsgroup_footer(bod)
            bod = strip_newsgroup_quoting(bod)
            features['body'][i] = bod

            prefix = 'Subject:'
            sub = ''
            for line in headers.split('\n'):
                if line.startswith(prefix):
                    sub = line[len(prefix):]
                    break
            features['subject'][i] = sub

        return features

pipeline = Pipeline([
    # Extract the subject & body
    ('subjectbody', SubjectBodyExtractor()),

    # Use FeatureUnion to combine the features from subject and body
    ('union', FeatureUnion(
        transformer_list=[

            # Pipeline for pulling features from the post's subject line
            ('subject', Pipeline([
                ('selector', ItemSelector(key='subject')),
                ('tfidf', TfidfVectorizer(min_df=50)),
            ])),

            # Pipeline for standard bag-of-words model for body
            ('body_bow', Pipeline([
                ('selector', ItemSelector(key='body')),
                ('tfidf', TfidfVectorizer()),
                ('best', TruncatedSVD(n_components=50)),
            ])),

            # Pipeline for pulling ad hoc features from post's body
            ('body_stats', Pipeline([
                ('selector', ItemSelector(key='body')),
                ('stats', TextStats()),  # returns a list of dicts
                ('vect', DictVectorizer()),  # list of dicts -> feature matrix
            ])),

        ],

        # weight components in FeatureUnion
        transformer_weights={
            'subject': 0.8,
            'body_bow': 0.5,
            'body_stats': 1.0,
        },
    )),

    # Use a SVC classifier on the combined features
    ('svc', SVC(kernel='linear')),
])

# limit the list of categories to make running this example faster.
categories = ['alt.atheism', 'talk.religion.misc']
train = fetch_20newsgroups(random_state=1,
                           subset='train',
                           categories=categories,
                           )
test = fetch_20newsgroups(random_state=1,
                          subset='test',
                          categories=categories,
                          )

pipeline.fit(train.data, train.target)
y = pipeline.predict(test.data)
print(classification_report(y, test.target))


#!/usr/bin/env python
# encoding: utf-8
"""
compute_transformation_error.py

Created by Maria Isabel Restrepo on 2012-09-24.
Copyright (c) 2012 . All rights reserved.
This script computes the distances betweeen an estimated similarity transformation and its ground truth
The transformation is used to transform a "source" coordinate system into a "target coordinate system"
To compute the error between the translations, the L2 norm diference translation vectors in the
"source coordinate system" is computed. Since distances are preserved under R and T, only scale is applied.
The rotation error is computed as the half angle between the normalized queternions i.e acos(|<q1,q2>|) in [0, pi/2]
"""
import os
import sys
import logging
import argparse
import vpcl_adaptor as vpcl
import numpy as np
from numpy import linalg as LA
import transformations as tf
import math
import matplotlib.pyplot as plt

sys.path.append(os.pardir)
import reg3d_transformations as reg3d_T

LOG = None

"""Compute the accuracy between the LIDAR fiducial points
and corresponding geo-register correspondances"""
def compute_ref_accuracy(fid_path, original_corrs_path,
                         geo_tform):

    #Load fiducial .ply
    fid = open(fid_path, 'r')
    fid_points = np.genfromtxt(fid, dtype=float, delimiter=' ',
                                skip_header=9)
    fid.close()
    #Load original corrs .ply
    fid = open(original_corrs_path, 'r')
    original_corrs = np.genfromtxt(fid, dtype=float,
                                    delimiter=' ', skip_header=9)
    fid.close()

    #Load transformation
    #************GEO**************"
    Tfis = open(geo_tform, 'r')
    lines = []
    lines = Tfis.readlines()
    scale_geo = float(lines[0])
    Ss_geo = tf.scale_matrix(scale_geo)
    quat_line = lines[1].split(" ")
    quat_geo = np.array([float(quat_line[3]), float(quat_line[0]),
                         float(quat_line[1]), float(quat_line[2])])
    Rs_geo = tf.quaternion_matrix(quat_geo)
    trans_line = lines[2].split(" ")
    trans_geo = np.array([float(trans_line[0]), float(trans_line[1]),
                          float(trans_line[2])])
    Tfis.close()
    Hs_geo = Rs_geo.copy()
    Hs_geo[:3, 3] = trans_geo[:3]
    Hs_geo = Ss_geo.dot(Hs_geo)

    LOG.debug("\n******Geo***** \n Scale: \n%s \nR:\n%s \nT:\n%s \nH:\n%s",
        Ss_geo, Rs_geo, trans_geo, Hs_geo)

    #Compute the "reference error"
    #i.e. fiducial points - geo registered correspondances
    npoints, c = fid_points.shape
    if npoints != 30:
        LOG.warn("Number of fiducial point is NOT 30")
    if c != 3:
        LOG.error("Fiducial points has the wrong number of dimensions")
    # import code; code.interact(local=locals())

    fid_points_hom = np.hstack((fid_points, np.ones([npoints, 1]))).T
    original_corrs_hom = np.hstack((original_corrs, np.ones([npoints, 1]))).T
    geo_corrs_hom = Hs_geo.dot(original_corrs_hom)
    geo_ref_diff = geo_corrs_hom - fid_points_hom

    # import pdb; pdb.set_trace()

    delta_z = np.sqrt(geo_ref_diff[2, :] * geo_ref_diff[2, :])
    delta_r = np.sqrt(geo_ref_diff[0, :] * geo_ref_diff[0, :] +
                      geo_ref_diff[1, :] * geo_ref_diff[1, :])

    return delta_z, delta_r

def compute_geo_accuracy(fid_path, original_corrs_path,
                         geo_tform, trials_root, desc_name,
                         niter, ntrials, percentile=99):

    #Load fiducial .ply
    fid = open(fid_path, 'r')
    fid_points = np.genfromtxt(fid, delimiter=' ',
                                skip_header=9)
    fid.close()

    #Load original corrs .ply
    fid = open(original_corrs_path, 'r')
    original_corrs = np.genfromtxt(fid, delimiter=' ', skip_header=9)
    fid.close()

    #load the geo tranformation
    GEO = reg3d_T.geo_transformation(geo_tform);

    #Compute the "reference error"
    #i.e. fiducial points - geo registered correspondances
    npoints, c = fid_points.shape
    if npoints != 30:
        LOG.warn("Number of fiducial point is NOT 30")
    if c != 3:
        LOG.error("Fiducial points has the wrong number of dimensions")
    # import code; code.interact(local=locals())

    fid_points_hom = np.hstack((fid_points, np.ones([npoints, 1]))).T
    original_corrs_hom = np.hstack((original_corrs, np.ones([npoints, 1]))).T
    geo_corrs_hom = GEO.transform_points(original_corrs_hom)
    geo_ref_diff = geo_corrs_hom - fid_points_hom

    # import pdb; pdb.set_trace()

    delta_z = (geo_ref_diff[2, :] **2) ** (1./2.)
    delta_r = (geo_ref_diff[0, :] **2 + geo_ref_diff[1, :] **2 )** (1./2.)

    delta_z_ia = np.zeros([ntrials, npoints])
    delta_r_ia = np.zeros([ntrials, npoints])
    delta_z_icp = np.zeros([ntrials, npoints])
    delta_r_icp = np.zeros([ntrials, npoints])

    for trial in range(0, ntrials):

        print "********Trial", trial, "**********"

        #Load the transformations for this trial

        #************Hs**************#
        #read source to target "Ground Truth" Transformation
        Tfile = trials_root + "/trial_" + str(trial) + "/Hs_inv.txt"
        GT_Tform = reg3d_T.gt_transformation(Tfile)

        src_features_dir = (trials_root + "/trial_" + str(trial) +
                            "/" + desc_name)

        Tfile_ia = (src_features_dir + "/ia_transformation_" +
                    str(percentile) + "_" + str(niter) + ".txt")
        Tfile_icp = (src_features_dir + "/icp_transformation_" +
                     str(percentile) + "_" + str(niter) + ".txt")

        REG_Tform = reg3d_T.pcl_transformation(Tfile_ia, Tfile_icp)
        Hs_ia_error = REG_Tform.Hs_ia.dot(GT_Tform.Hs)
        Hs_icp_error = REG_Tform.Hs_icp.dot(GT_Tform.Hs)

        # transform the points with the residual transformations
        ia_corrs_hom = Hs_ia_error.dot(original_corrs_hom)
        icp_corrs_hom = Hs_icp_error.dot(original_corrs_hom)
        # geo-register
        geo_ia_corrs_hom = GEO.transform_points(ia_corrs_hom)
        geo_icp_corrs_hom = GEO.transform_points(icp_corrs_hom)
        # distances
        geo_ia_ref_diff = geo_ia_corrs_hom - fid_points_hom
        geo_icp_ref_diff = geo_icp_corrs_hom - fid_points_hom

        delta_z_ia[trial, :] = np.sqrt(geo_ia_ref_diff[2, :] ** 2)
        delta_r_ia[trial, :] = np.sqrt(geo_ia_ref_diff[0, :] ** 2 +
                                       geo_ia_ref_diff[1, :] ** 2 )

        delta_z_icp[trial, :] = np.sqrt(geo_icp_ref_diff[2, :] ** 2)
        delta_r_icp[trial, :] = np.sqrt(geo_icp_ref_diff[0, :] ** 2 +
                                        geo_icp_ref_diff[1, :] ** 2)

        # import pdb; pdb.set_trace()

    return delta_z, delta_r,\
           delta_z_ia, delta_r_ia, \
           delta_z_icp, delta_r_icp

def main(logfile=None):

    global LOG
    LOG = setlogging(logfile)

    descriptors = ["FPFH_30", "SHOT_30"]
    niter = 500;
    ntrials = 10;
    plot_errors = True;
    if (plot_errors):
      colors = ['magenta','green'];
      markers = ['o', 's', '*', '+', '^', 'v']
    fid_path = "/data/lidar_providence/downtown_offset-1-financial-dan-pts1.ply"
    original_corrs_path = "/data/lidar_providence/downtown_offset-1-financial-dan-pts0.ply"
    trials_root = "/Users/isa/Experiments/reg3d_eval/downtown_dan";
    geo_tform = "/data/lidar_providence/downtown_offset-1-financial-dan-Hs.txt"
    for d_idx in range(0, len(descriptors)):
        desc_name = descriptors[d_idx]

        delta_z, delta_r, \
        delta_z_ia, delta_r_ia, \
        delta_z_icp, delta_r_icp = compute_geo_accuracy(fid_path,
                                    original_corrs_path,
                                    geo_tform, trials_root, desc_name,
                                    niter, ntrials)

        #sort errors for all trials to get the 70 80 90 % errors
        delta_z_ia.sort(axis=0)
        delta_r_ia.sort(axis=0)

        delta_z_icp.sort(axis=0)
        delta_r_icp.sort(axis=0)

        CE_70_ia = delta_r_ia[int(0.7 * ntrials) - 1, :]
        CE_80_ia = delta_r_ia[int(0.8 * ntrials) - 1, :]
        CE_90_ia = delta_r_ia[int(0.9 * ntrials) - 1, :]

        LE_70_ia = delta_z_ia[int(0.7 * ntrials) - 1, :]
        LE_80_ia = delta_z_ia[int(0.8 * ntrials) - 1, :]
        LE_90_ia = delta_z_ia[int(0.9 * ntrials) - 1, :]

        CE_70_icp = delta_r_icp[int(0.7 * ntrials) - 1, :]
        CE_80_icp = delta_r_icp[int(0.8 * ntrials) - 1, :]
        CE_90_icp = delta_r_icp[int(0.9 * ntrials) - 1, :]

        LE_70_icp = delta_z_icp[int(0.7 * ntrials) - 1, :]
        LE_80_icp = delta_z_icp[int(0.8 * ntrials) - 1, :]
        LE_90_icp = delta_z_icp[int(0.9 * ntrials) - 1, :]

        if (plot_errors):
        #Plot CE and LE
            fig_ia_CE = plt.figure()
            ax_ia_CE = fig_ia_CE.add_subplot(111);
            plt.hold(True);
            plt.axis(tight=True);
            ax_ia_CE.plot(CE_70_ia, "--s", color="green", label= "CE_70");
            ax_ia_CE.plot(CE_80_ia, "--^", color="magenta", label= "CE_80");
            ax_ia_CE.plot(CE_90_ia, "--*", color="blue", label= "CE_90");
            ax_ia_CE.plot( delta_r, "--o", color="cyan", label=  "GT");
            ax_ia_CE.set_xlabel('Fiducial Marker (index)',fontsize= 20);
            ax_ia_CE.set_ylabel('Error (meters)',fontsize= 20);
            ax_ia_CE.legend(loc='best', frameon=False);
            # ax_ia_CE.set_title('IA CE')
            fname = trials_root + "/GEO_results/IA_CE_" + desc_name + ".pdf"
            fig_ia_CE.savefig(fname, transparent=True, pad_inches=5)

            fig_ia_LE = plt.figure()
            ax_ia_LE = fig_ia_LE.add_subplot(111);
            plt.hold(True);
            plt.axis(tight=True);
            ax_ia_LE.plot(LE_70_ia, "--s", color="green", label= "LE_70");
            ax_ia_LE.plot(LE_80_ia, "--^", color="magenta", label= "LE_80");
            ax_ia_LE.plot(LE_90_ia, "--*", color="blue", label= "LE_90");
            ax_ia_LE.plot( delta_z, "--o", color="cyan", label=  "GT");
            ax_ia_LE.set_xlabel('Fiducial Marker (index)',fontsize= 20);
            ax_ia_LE.set_ylabel('Error (meters)',fontsize= 20);
            ax_ia_LE.legend(loc='best', frameon=False);
            # ax_ia_LE.set_title('IA LE')
            fname = trials_root + "/GEO_results/IA_LE_" + desc_name + ".pdf"
            fig_ia_LE.savefig(fname, transparent=True, pad_inches=5)

            fig_icp_CE = plt.figure()
            ax_icp_CE = fig_icp_CE.add_subplot(111);
            plt.hold(True);
            plt.axis(tight=True);
            ax_icp_CE.plot(CE_70_icp, "--s", color="green", label= "CE_70");
            ax_icp_CE.plot(CE_80_icp, "--^", color="magenta", label= "CE_80");
            ax_icp_CE.plot(CE_90_icp, "--*", color="blue", label= "CE_90");
            ax_icp_CE.plot( delta_r, "--o", color="cyan", label=  "GT");
            ax_icp_CE.set_xlabel('Fiducial Marker (index)',fontsize= 20);
            ax_icp_CE.set_ylabel('Error (meters)',fontsize= 20);
            ax_icp_CE.legend(loc='best', frameon=False);
            # ax_icp_CE.set_title('ICP CE')
            fname = trials_root + "/GEO_results/ICP_CE_" + desc_name + ".pdf"
            fig_icp_CE.savefig(fname, transparent=True, pad_inches=5)

            fig_icp_LE = plt.figure()
            ax_icp_LE = fig_icp_LE.add_subplot(111);
            plt.hold(True);
            plt.axis(tight=True);
            ax_icp_LE.plot(LE_70_icp, "--s", color="green", label= "LE_70");
            ax_icp_LE.plot(LE_80_icp, "--^", color="magenta", label= "LE_80");
            ax_icp_LE.plot(LE_90_icp, "--*", color="blue", label= "LE_90");
            ax_icp_LE.plot( delta_z, "--o", color="cyan", label=  "GT");
            ax_icp_LE.set_xlabel('Fiducial Marker (index)',fontsize= 20);
            ax_icp_LE.set_ylabel('Error (meters)',fontsize= 20);
            ax_icp_LE.legend(loc='best', frameon=False);
            # ax_icp_LE.set_title('ICP LE')
            fname = trials_root + "/GEO_results/ICP_LE_" + desc_name + ".pdf"
            fig_icp_LE.savefig(fname, transparent=True, pad_inches=5)

            # axT.set_xlim((0,505) );
            # axT.set_yticks(np.arange(0.0,250.0,20));
            # # axT.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
            # #             ncol=4, mode="expand", borderaxespad=0.)
            #
            # figT.savefig("/Users/isa/Experiments/reg3d_eval/downtown_dan/T_error.pdf", transparent=True, pad_inches=5)

            # plt.show();

    # import pdb; pdb.set_trace()

def setlogging(logfile=None):
    level = logging.DEBUG
    logger = logging.getLogger(__name__)
    logger.setLevel(level)
    # create formatter and add it to the handlers
    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(level)
    ch.setFormatter(formatter)
    # add the handlers to logger
    logger.addHandler(ch)

    # create file handler which logs error messages
    if logfile:
        print "Logging to file"
        fh = logging.FileHandler(logfile)
        fh.setLevel(level)
        fh.setFormatter(formatter)
        logger.addHandler(fh)

     #test logging
    logger.debug("debug message")
    logger.info("info message")
    logger.warn("warn message")
    logger.error("error message")
    logger.critical("critical message")

    return logger

if __name__ == '__main__':

    # initialize the parser object:
    parser = argparse.ArgumentParser(description="Export PLY to PCD file")

    # define options here:
    parser.add_argument("-v", "--verbose",      action='store',    type = bool,   dest="verbose",   default=True,  help="Write debug log to log_file")
    parser.add_argument("-L", "--log", dest="logfile", help="write debug log to log_file")

    args = parser.parse_args(argv)

    # set up logging
    if args.verbose:
        status = main(args.logfile)
    else:
        status = main()

    sys.exit(status)


"""
=============================================
A demo of the Spectral Biclustering algorithm
=============================================

This example demonstrates how to generate a checkerboard dataset and
bicluster it using the Spectral Biclustering algorithm.

The data is generated with the ``make_checkerboard`` function, then
shuffled and passed to the Spectral Biclustering algorithm. The rows
and columns of the shuffled matrix are rearranged to show the
biclusters found by the algorithm.

The outer product of the row and column label vectors shows a
representation of the checkerboard structure.

"""
print(__doc__)

# Author: Kemal Eren <kemal@kemaleren.com>
# License: BSD 3 clause

import numpy as np
from matplotlib import pyplot as plt

from sklearn.datasets import make_checkerboard
from sklearn.datasets import samples_generator as sg
from sklearn.cluster.bicluster import SpectralBiclustering
from sklearn.metrics import consensus_score

n_clusters = (4, 3)
data, rows, columns = make_checkerboard(
    shape=(300, 300), n_clusters=n_clusters, noise=10,
    shuffle=False, random_state=0)

plt.matshow(data, cmap=plt.cm.Blues)
plt.title("Original dataset")

data, row_idx, col_idx = sg._shuffle(data, random_state=0)
plt.matshow(data, cmap=plt.cm.Blues)
plt.title("Shuffled dataset")

model = SpectralBiclustering(n_clusters=n_clusters, method='log',
                             random_state=0)
model.fit(data)
score = consensus_score(model.biclusters_,
                        (rows[:, row_idx], columns[:, col_idx]))

print("consensus score: {:.1f}".format(score))

fit_data = data[np.argsort(model.row_labels_)]
fit_data = fit_data[:, np.argsort(model.column_labels_)]

plt.matshow(fit_data, cmap=plt.cm.Blues)
plt.title("After biclustering; rearranged to show biclusters")

plt.matshow(np.outer(np.sort(model.row_labels_) + 1,
                     np.sort(model.column_labels_) + 1),
            cmap=plt.cm.Blues)
plt.title("Checkerboard structure of rearranged data")

plt.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 11 10:10:24 2017

@author: Mark
"""
import numpy as np
import matplotlib.pyplot as plt

#Define functions
def model(state_0,A,B):
    state_1 = A*state_0 +  np.random.normal(0,B)
    return state_1

state_null=np.random.normal(0,0.4)

def observation_function(state,R):
    obs=state+np.random.normal(0,R)
    return obs

def forecast(state_0,cov_0,A,B):
    state_1=A*state_0
    cov_1=A*cov_0*A+B
    return state_1,cov_1

def analysis_formulas(state_1_hat,cov_1_hat,K,H,obs_0):
    state_1 = state_1_hat - K*(H*state_1_hat - obs_0)
    cov_1 = cov_1_hat - K*H*cov_1_hat
    return state_1, cov_1

def kalman_gain(cov_1_hat,H,R):
    K = cov_1_hat*H*(R+H*cov_1_hat*H)**(-1)
    return K

#Initialize model parameters
A = 0.5
H = 1
B = 0.5
R = 0.1
lev = 100

#Sythetic Model
STATE_real = np.zeros(lev)
OBS_real = np.zeros(lev)
STATE_real[0] = np.random.normal(5,0.1)
OBS_real[0] = observation_function(STATE_real[0],R)
for i in range (1,lev-1):
    STATE_real[i] = model(STATE_real[i-1],0.4,0.01)
    OBS_real[i] =  observation_function(STATE_real[i],R)

#Kalman-filter
STATE = np.zeros(lev)
COV = np.zeros(lev)
STATE[0] = state_null
COV[0] = B

for i in range (1,lev-1):
    (state_hat,cov_hat) = forecast(STATE[i-1],COV[i-1],A,B)
    K = kalman_gain(cov_hat,H,R)
    (STATE[i],COV[i]) = analysis_formulas(state_hat,cov_hat,K,H,OBS_real[i])

plt.plot(STATE)
plt.plot(STATE_real)


#!/usr/bin/env perl

# Find all contacts beween domains..

import sys, os, re, string
import argparse
from os.path import expanduser

home = expanduser("~")
sys.path.append(home + '/bioinfo-toolbox/parsing')
sys.path.append(home + '/git/bioinfo-toolbox/parsing')

import parse_contacts
import numpy as np
import matplotlib
matplotlib.use('Agg')

sep=5

contacts = parse_contacts.parse(open(c_filename, 'r'), sep)
contacts_np = parse_contacts.get_numpy_cmap(contacts)
contacts_np = contacts_np[start:end,start:end]

for i in range(len(contacts)):
    score = contacts[i][0]
    c_x = contacts[i][1] - 1
    c_y = contacts[i][2] - 1

    # only look at contacts within given range
    # default: take full sequence range into account
    if c_x < start or c_x >= end:
        continue
    if c_y < start or c_y >= end:
        continue
    if c_y-c_x < start or c_y >= end:
        continue

    if c_x < domain

        pos_diff = abs(c_x - c_y)
        too_close = pos_diff < 5

if __name__ == "__main__":

    p = argparse.ArgumentParser(description='Plot protein residue contact maps.')
    p.add_argument('-t', '--threshold', default=-1, type=float)
    p.add_argument('--start', default=0, type=int)
    p.add_argument('--end', default=-1, type=int)
    p.add_argument('--sep', default=5, type=int)
    p.add_argument('--domain', default=-1, type=int)


"""
plotable rich object display on IPython/Jupyter notebooks
"""

__author__ = "Philippe Guglielmetti"
__copyright__ = "Copyright 2015, Philippe Guglielmetti"
__credits__ = []
__license__ = "LGPL"

# import matplotlib and set backend once for all
from . import itertools2
import os
import io
import sys
import logging
import base64
import matplotlib

if os.getenv('TRAVIS'):  # are we running https://travis-ci.org/ automated tests ?
    matplotlib.use('Agg')  # Force matplotlib  not to use any Xwindows backend
elif sys.gettrace():  # http://stackoverflow.com/questions/333995/how-to-detect-that-python-code-is-being-executed-through-the-debugger
    matplotlib.use('Agg')  # because 'QtAgg' crashes python while debugging
else:
    pass
    # matplotlib.use('pdf') #for high quality pdf, but doesn't work for png, svg ...

logging.info('matplotlib backend is %s' % matplotlib.get_backend())

class Plot(object):
    """base class for plotable rich object display on IPython notebooks
    inspired from http://nbviewer.ipython.org/github/ipython/ipython/blob/3607712653c66d63e0d7f13f073bde8c0f209ba8/docs/examples/notebooks/display_protocol.ipynb
    """

    def _plot(self, ax, **kwargs):
        """abstract method, must be overriden

        :param ax: `matplotlib.axis`
        :return ax: `matplotlib.axis` after plot
        """
        raise NotImplementedError(
            'objects derived from plot.PLot must define a _plot method')
        return ax

    def render(self, fmt='svg', **kwargs):
        return render([self], fmt, **kwargs)  # call global function

    def save(self, filename, **kwargs):
        return save([self], filename, **kwargs)  # call global function

    # for IPython notebooks

    def _repr_html_(self):
        """default rich format is svg plot"""
        try:
            return self._repr_svg_()
        except NotImplementedError:
            pass
        # this returns  the same as _repr_png_, but is Table compatible
        buffer = self.render('png')
        s = base64.b64encode(buffer).decode('utf-8')
        return '<img src="data:image/png;base64,%s">' % s

    def html(self, **kwargs):
        from IPython.display import HTML
        return HTML(self._repr_html_(**kwargs))

    def svg(self, **kwargs):
        from IPython.display import SVG
        return SVG(self._repr_svg_(**kwargs))

    def _repr_svg_(self, **kwargs):
        return self.render(fmt='svg', **kwargs).decode('utf-8')

    def png(self, **kwargs):
        from IPython.display import Image
        return Image(self._repr_png_(**kwargs), embed=True)

    def _repr_png_(self, **kwargs):
        return self.render(fmt='png', **kwargs)

    def plot(self, **kwargs):
        """ renders on IPython Notebook
        (alias to make usage more straightforward)
        """
        return self.svg(**kwargs)

def render(plotables, fmt='svg', **kwargs):
    """renders several Plot objects"""
    import matplotlib.pyplot as plt

    # extract optional arguments used for rasterization
    printargs, kwargs = itertools2.dictsplit(
        kwargs,
        ['dpi', 'transparent', 'facecolor', 'background', 'figsize']
    )

    ylim = kwargs.pop('ylim', None)
    xlim = kwargs.pop('xlim', None)
    title = kwargs.pop('title', None)

    fig, ax = plt.subplots()

    labels = kwargs.pop('labels', [None] * len(plotables))
    # slightly shift the points to make superimposed curves more visible
    offset = kwargs.pop('offset', 0)

    for i, obj in enumerate(plotables):
        if labels[i] is None:
            labels[i] = str(obj)
        if not title:
            try:
                title = obj._repr_latex_()
                # check that title can be used in matplotlib
                from matplotlib.mathtext import MathTextParser
                parser = MathTextParser('path').parse(title)
            except Exception as e:
                title = labels[i]
        ax = obj._plot(ax, label=labels[i], offset=i * offset, **kwargs)

    if ylim:
        plt.ylim(ylim)
    if xlim:
        plt.xlim(xlim)

    ax.set_title(title)

    if len(labels) > 1:
        ax.legend()

    output = io.BytesIO()
    fig.savefig(output, format=fmt, **printargs)
    data = output.getvalue()
    plt.close(fig)
    return data

def png(plotables, **kwargs):
    from IPython.display import Image
    return Image(render(plotables, 'png', **kwargs), embed=True)

def svg(plotables, **kwargs):
    from IPython.display import SVG
    return SVG(render(plotables, 'svg', **kwargs))

plot = svg

def save(plotables, filename, **kwargs):
    ext = filename.split('.')[-1].lower()
    kwargs.setdefault('dpi', 600)  # force good quality
    return open(filename, 'wb').write(render(plotables, ext, **kwargs))


# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Implementations of different data feeders to provide data for TF trainer."""

# TODO(ipolosukhin): Replace this module with feed-dict queue runners & queues.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import itertools
import math

import numpy as np
import six
from six.moves import xrange  # pylint: disable=redefined-builtin

from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.platform import tf_logging as logging

# pylint: disable=g-multiple-import,g-bad-import-order
from .pandas_io import HAS_PANDAS, extract_pandas_data, extract_pandas_matrix, extract_pandas_labels
from .dask_io import HAS_DASK, extract_dask_data, extract_dask_labels

# pylint: enable=g-multiple-import,g-bad-import-order

def _get_in_out_shape(x_shape, y_shape, n_classes, batch_size=None):
  """Returns shape for input and output of the data feeder."""
  x_is_dict, y_is_dict = isinstance(
      x_shape, dict), y_shape is not None and isinstance(y_shape, dict)
  if y_is_dict and n_classes is not None:
    assert (isinstance(n_classes, dict))

  if batch_size is None:
    batch_size = list(x_shape.values())[0][0] if x_is_dict else x_shape[0]
  elif batch_size <= 0:
    raise ValueError('Invalid batch_size %d.' % batch_size)

  if x_is_dict:
    input_shape = {}
    for k, v in list(x_shape.items()):
      input_shape[k] = [batch_size] + (list(v[1:]) if len(v) > 1 else [1])
  else:
    x_shape = list(x_shape[1:]) if len(x_shape) > 1 else [1]
    input_shape = [batch_size] + x_shape

  if y_shape is None:
    return input_shape, None, batch_size

  def out_el_shape(out_shape, num_classes):
    out_shape = list(out_shape[1:]) if len(out_shape) > 1 else []
    # Skip first dimension if it is 1.
    if out_shape and out_shape[0] == 1:
      out_shape = out_shape[1:]
    if num_classes is not None and num_classes > 1:
      return [batch_size] + out_shape + [num_classes]
    else:
      return [batch_size] + out_shape

  if not y_is_dict:
    output_shape = out_el_shape(y_shape, n_classes)
  else:
    output_shape = dict([
        (k, out_el_shape(v, n_classes[k]
                         if n_classes is not None and k in n_classes else None))
        for k, v in list(y_shape.items())
    ])

  return input_shape, output_shape, batch_size

def _data_type_filter(x, y):
  """Filter data types into acceptable format."""
  if HAS_DASK:
    x = extract_dask_data(x)
    if y is not None:
      y = extract_dask_labels(y)
  if HAS_PANDAS:
    x = extract_pandas_data(x)
    if y is not None:
      y = extract_pandas_labels(y)
  return x, y

def _is_iterable(x):
  return hasattr(x, 'next') or hasattr(x, '__next__')

def setup_train_data_feeder(x,
                            y,
                            n_classes,
                            batch_size=None,
                            shuffle=True,
                            epochs=None):
  """Create data feeder, to sample inputs from dataset.

  If `x` and `y` are iterators, use `StreamingDataFeeder`.

  Args:
    x: numpy, pandas or Dask matrix or dictionary of aforementioned. Also
      supports iterables.
    y: numpy, pandas or Dask array or dictionary of aforementioned. Also
      supports
      iterables.
    n_classes: number of classes. Must be None or same type as y. In case, `y`
      is `dict`
      (or iterable which returns dict) such that `n_classes[key] = n_classes for
        y[key]`
    batch_size: size to split data into parts. Must be >= 1.
    shuffle: Whether to shuffle the inputs.
    epochs: Number of epochs to run.

  Returns:
    DataFeeder object that returns training data.

  Raises:
    ValueError: if one of `x` and `y` is iterable and the other is not.
  """
  x, y = _data_type_filter(x, y)
  if HAS_DASK:
    # pylint: disable=g-import-not-at-top
    import dask.dataframe as dd
    if (isinstance(x, (dd.Series, dd.DataFrame)) and
        (y is None or isinstance(y, (dd.Series, dd.DataFrame)))):
      data_feeder_cls = DaskDataFeeder
    else:
      data_feeder_cls = DataFeeder
  else:
    data_feeder_cls = DataFeeder

  if _is_iterable(x):
    if y is not None and not _is_iterable(y):
      raise ValueError('Both x and y should be iterators for '
                       'streaming learning to work.')
    return StreamingDataFeeder(x, y, n_classes, batch_size)
  return data_feeder_cls(
      x, y, n_classes, batch_size, shuffle=shuffle, epochs=epochs)

def _batch_data(x, batch_size=None):
  if (batch_size is not None) and (batch_size <= 0):
    raise ValueError('Invalid batch_size %d.' % batch_size)

  x_first_el = six.next(x)
  x = itertools.chain([x_first_el], x)

  chunk = dict([(k, []) for k in list(x_first_el.keys())]) if isinstance(
      x_first_el, dict) else []
  chunk_filled = False
  for data in x:
    if isinstance(data, dict):
      for k, v in list(data.items()):
        chunk[k].append(v)
        if (batch_size is not None) and (len(chunk[k]) >= batch_size):
          chunk[k] = np.matrix(chunk[k])
          chunk_filled = True
      if chunk_filled:
        yield chunk
        chunk = dict([(k, []) for k in list(x_first_el.keys())]) if isinstance(
            x_first_el, dict) else []
        chunk_filled = False
    else:
      chunk.append(data)
      if (batch_size is not None) and (len(chunk) >= batch_size):
        yield np.matrix(chunk)
        chunk = []

  if isinstance(x_first_el, dict):
    for k, v in list(data.items()):
      chunk[k] = np.matrix(chunk[k])
    yield chunk
  else:
    yield np.matrix(chunk)

def setup_predict_data_feeder(x, batch_size=None):
  """Returns an iterable for feeding into predict step.

  Args:
    x: numpy, pandas, Dask array or dictionary of aforementioned. Also supports
      iterable.
    batch_size: Size of batches to split data into. If `None`, returns one
      batch of full size.

  Returns:
    List or iterator (or dictionary thereof) of parts of data to predict on.

  Raises:
    ValueError: if `batch_size` <= 0.
  """
  if HAS_DASK:
    x = extract_dask_data(x)
  if HAS_PANDAS:
    x = extract_pandas_data(x)
  if _is_iterable(x):
    return _batch_data(x, batch_size)
  if len(x.shape) == 1:
    x = np.reshape(x, (-1, 1))
  if batch_size is not None:
    if batch_size <= 0:
      raise ValueError('Invalid batch_size %d.' % batch_size)
    n_batches = int(math.ceil(float(len(x)) / batch_size))
    return [x[i * batch_size:(i + 1) * batch_size] for i in xrange(n_batches)]
  return [x]

def setup_processor_data_feeder(x):
  """Sets up processor iterable.

  Args:
    x: numpy, pandas or iterable.

  Returns:
    Iterable of data to process.
  """
  if HAS_PANDAS:
    x = extract_pandas_matrix(x)
  return x

def check_array(array, dtype):
  """Checks array on dtype and converts it if different.

  Args:
    array: Input array.
    dtype: Expected dtype.

  Returns:
    Original array or converted.
  """
  # skip check if array is instance of other classes, e.g. h5py.Dataset
  # to avoid copying array and loading whole data into memory
  if isinstance(array, (np.ndarray, list)):
    array = np.array(array, dtype=dtype, order=None, copy=False)
  return array

def _access(data, iloc):
  """Accesses an element from collection, using integer location based indexing.

  Args:
    data: array-like. The collection to access
    iloc: `int` or `list` of `int`s. Location(s) to access in `collection`

  Returns:
    The element of `a` found at location(s) `iloc`.
  """
  if HAS_PANDAS:
    import pandas as pd  # pylint: disable=g-import-not-at-top
    if isinstance(data, pd.Series) or isinstance(data, pd.DataFrame):
      return data.iloc[iloc]
  return data[iloc]

def _check_dtype(dtype):
  if dtypes.as_dtype(dtype) == dtypes.float64:
    logging.warn(
        'float64 is not supported by many models, consider casting to float32.')
  return dtype

class DataFeeder(object):
  """Data feeder is an example class to sample data for TF trainer."""

  def __init__(self,
               x,
               y,
               n_classes,
               batch_size=None,
               shuffle=True,
               random_state=None,
               epochs=None):
    """Initializes a DataFeeder instance.

    Args:
      x: One feature sample which can either Nd numpy matrix of shape
        `[n_samples, n_features, ...]` or dictionary of Nd numpy matrix.
      y: label vector, either floats for regression or class id for
        classification. If matrix, will consider as a sequence of labels.
        Can be `None` for unsupervised setting. Also supports dictionary of
        labels.
      n_classes: Number of classes, 0 and 1 are considered regression, `None`
        will pass through the input labels without one-hot conversion. Also, if
        `y` is `dict`, then `n_classes` must be `dict` such that
        `n_classes[key] = n_classes for label y[key]`, `None` otherwise.
      batch_size: Mini-batch size to accumulate samples in one mini batch.
      shuffle: Whether to shuffle `x`.
      random_state: Numpy `RandomState` object to reproduce sampling.
      epochs: Number of times to iterate over input data before raising
        `StopIteration` exception.

    Attributes:
      x: Input features (ndarray or dictionary of ndarrays).
      y: Input label (ndarray or dictionary of ndarrays).
      n_classes: Number of classes (if `None`, pass through indices without
        one-hot conversion).
      batch_size: Mini-batch size to accumulate.
      input_shape: Shape of the input (or dictionary of shapes).
      output_shape: Shape of the output (or dictionary of shapes).
      input_dtype: DType of input (or dictionary of shapes).
      output_dtype: DType of output (or dictionary of shapes.
    """
    x_is_dict, y_is_dict = isinstance(x, dict), y is not None and isinstance(
        y, dict)
    if isinstance(y, list):
      y = np.array(y)

    self._x = dict([(k, check_array(v, v.dtype)) for k, v in list(x.items())
                   ]) if x_is_dict else check_array(x, x.dtype)
    self._y = None if y is None else \
      dict([(k, check_array(v, v.dtype)) for k, v in list(y.items())]) if x_is_dict else check_array(y, y.dtype)

    # self.n_classes is not None means we're converting raw target indices to one-hot.
    if n_classes is not None:
      if not y_is_dict:
        y_dtype = (np.int64
                   if n_classes is not None and n_classes > 1 else np.float32)
        self._y = (None if y is None else check_array(y, dtype=y_dtype))

    self.n_classes = n_classes
    self.max_epochs = epochs

    x_shape = dict([(k, v.shape) for k, v in list(self._x.items())
                   ]) if x_is_dict else self._x.shape
    y_shape = dict([(k, v.shape) for k, v in list(self._y.items())
                   ]) if y_is_dict else None if y is None else self._y.shape

    self.input_shape, self.output_shape, self._batch_size = _get_in_out_shape(
        x_shape, y_shape, n_classes, batch_size)

    # Input dtype matches dtype of x.
    self._input_dtype = dict([(k, _check_dtype(v.dtype)) for k, v in list(self._x.items())]) if x_is_dict \
      else _check_dtype(self._x.dtype)

    # note: self._output_dtype = np.float32 when y is None
    self._output_dtype = dict([(k, _check_dtype(v.dtype)) for k, v in list(self._y.items())]) if y_is_dict \
      else _check_dtype(self._y.dtype) if y is not None else np.float32

    # self.n_classes is None means we're passing in raw target indices
    if n_classes is not None and y_is_dict:
      for key in list(n_classes.keys()):
        if key in self._output_dtype:
          self._output_dtype[key] = np.float32

    self._shuffle = shuffle
    self.random_state = np.random.RandomState(
        42) if random_state is None else random_state

    num_samples = list(self._x.values())[0].shape[
        0] if x_is_dict else self._x.shape[0]
    if self._shuffle:
      self.indices = self.random_state.permutation(num_samples)
    else:
      self.indices = np.array(range(num_samples))
    self.offset = 0
    self.epoch = 0
    self._epoch_placeholder = None

  @property
  def x(self):
    return self._x

  @property
  def y(self):
    return self._y

  @property
  def shuffle(self):
    return self._shuffle

  @property
  def input_dtype(self):
    return self._input_dtype

  @property
  def output_dtype(self):
    return self._output_dtype

  @property
  def batch_size(self):
    return self._batch_size

  def make_epoch_variable(self):
    """Adds a placeholder variable for the epoch to the graph.

    Returns:
      The epoch placeholder.
    """
    self._epoch_placeholder = array_ops.placeholder(
        dtypes.int32, [1], name='epoch')
    return self._epoch_placeholder

  def input_builder(self):
    """Builds inputs in the graph.

    Returns:
      Two placeholders for inputs and outputs.
    """

    def get_placeholder(shape, dtype, name_prepend):
      if shape is None:
        return None
      if isinstance(shape, dict):
        placeholder = {}
        for key in list(shape.keys()):
          placeholder[key] = array_ops.placeholder(
              dtypes.as_dtype(dtype[key]), [None] + shape[key][1:],
              name=name_prepend + '_' + key)
      else:
        placeholder = array_ops.placeholder(
            dtypes.as_dtype(dtype), [None] + shape[1:], name=name_prepend)
      return placeholder

    self._input_placeholder = get_placeholder(self.input_shape,
                                              self._input_dtype, 'input')
    self._output_placeholder = get_placeholder(self.output_shape,
                                               self._output_dtype, 'output')
    return self._input_placeholder, self._output_placeholder

  def set_placeholders(self, input_placeholder, output_placeholder):
    """Sets placeholders for this data feeder.

    Args:
      input_placeholder: Placeholder for `x` variable. Should match shape
        of the examples in the x dataset.
      output_placeholder: Placeholder for `y` variable. Should match
        shape of the examples in the y dataset. Can be `None`.
    """
    self._input_placeholder = input_placeholder
    self._output_placeholder = output_placeholder

  def get_feed_params(self):
    """Function returns a `dict` with data feed params while training.

    Returns:
      A `dict` with data feed params while training.
    """
    return {
        'epoch': self.epoch,
        'offset': self.offset,
        'batch_size': self._batch_size
    }

  def get_feed_dict_fn(self):
    """Returns a function that samples data into given placeholders.

    Returns:
      A function that when called samples a random subset of batch size
      from `x` and `y`.
    """
    x_is_dict, y_is_dict = isinstance(
        self._x, dict), self._y is not None and isinstance(self._y, dict)

    # Assign input features from random indices.
    def extract(data, indices):
      return (np.array(_access(data, indices)).reshape((indices.shape[0], 1)) if
              len(data.shape) == 1 else _access(data, indices))

    # assign labels from random indices
    def assign_label(data, shape, dtype, n_classes, indices):
      shape[0] = indices.shape[0]
      out = np.zeros(shape, dtype=dtype)
      for i in xrange(out.shape[0]):
        sample = indices[i]
        # self.n_classes is None means we're passing in raw target indices
        if n_classes is None:
          out[i] = _access(data, sample)
        else:
          if n_classes > 1:
            if len(shape) == 2:
              out.itemset((i, int(_access(data, sample))), 1.0)
            else:
              for idx, value in enumerate(_access(data, sample)):
                out.itemset(tuple([i, idx, value]), 1.0)
          else:
            out[i] = _access(data, sample)
      return out

    def _feed_dict_fn():
      """Function that samples data into given placeholders."""
      if self.max_epochs is not None and self.epoch + 1 > self.max_epochs:
        raise StopIteration
      assert self._input_placeholder is not None
      feed_dict = {}
      if self._epoch_placeholder is not None:
        feed_dict[self._epoch_placeholder.name] = [self.epoch]

      # Take next batch of indices.
      x_len = list(self._x.values())[0].shape[
          0] if x_is_dict else self._x.shape[0]
      end = min(x_len, self.offset + self._batch_size)
      batch_indices = self.indices[self.offset:end]

      # adding input placeholder
      feed_dict.update(
          dict([(self._input_placeholder[k].name, extract(v, batch_indices))
                for k, v in list(self._x.items())]) if x_is_dict else
          {self._input_placeholder.name: extract(self._x, batch_indices)})

      # move offset and reset it if necessary
      self.offset += self._batch_size
      if self.offset >= x_len:
        self.indices = self.random_state.permutation(
            x_len) if self._shuffle else np.array(range(x_len))
        self.offset = 0
        self.epoch += 1

      # return early if there are no labels
      if self._output_placeholder is None:
        return feed_dict

      # adding output placeholders
      if y_is_dict:
        for k, v in list(self._y.items()):
          n_classes = (self.n_classes[k] if k in self.n_classes else
                       None) if self.n_classes is not None else None
          shape, dtype = self.output_shape[k], self._output_dtype[k]
          feed_dict.update({
              self._output_placeholder[k].name:
                  assign_label(v, shape, dtype, n_classes, batch_indices)
          })
      else:
        shape, dtype, n_classes = self.output_shape, self._output_dtype, self.n_classes
        feed_dict.update({
            self._output_placeholder.name:
                assign_label(self._y, shape, dtype, n_classes, batch_indices)
        })

      return feed_dict

    return _feed_dict_fn

class StreamingDataFeeder(DataFeeder):
  """Data feeder for TF trainer that reads data from iterator.

  Streaming data feeder allows to read data as it comes it from disk or
  somewhere else. It's custom to have this iterators rotate infinetly over
  the dataset, to allow control of how much to learn on the trainer side.
  """

  def __init__(self, x, y, n_classes, batch_size):
    """Initializes a StreamingDataFeeder instance.

    Args:
      x: iterator each element of which returns one feature sample. Sample can
        be a Nd numpy matrix or dictionary of Nd numpy matrices.
      y: iterator each element of which returns one label sample. Sample can be
        a Nd numpy matrix or dictionary of Nd numpy matrices with 1 or many
        classes regression values.
      n_classes: indicator of how many classes the corresponding label sample
        has for the purposes of one-hot conversion of label. In case where `y`
        is a dictionary, `n_classes` must be dictionary (with same keys as `y`)
        of how many classes there are in each label in `y`. If key is
        present in `y` and missing in `n_classes`, the value is assumed `None`
        and no one-hot conversion will be applied to the label with that key.
      batch_size: Mini batch size to accumulate samples in one batch. If set
        `None`, then assumes that iterator to return already batched element.

    Attributes:
      x: input features (or dictionary of input features).
      y: input label (or dictionary of output features).
      n_classes: number of classes.
      batch_size: mini batch size to accumulate.
      input_shape: shape of the input (can be dictionary depending on `x`).
      output_shape: shape of the output (can be dictionary depending on `y`).
      input_dtype: dtype of input (can be dictionary depending on `x`).
      output_dtype: dtype of output (can be dictionary depending on `y`).
    """
    # pylint: disable=invalid-name,super-init-not-called
    x_first_el = six.next(x)
    self._x = itertools.chain([x_first_el], x)
    if y is not None:
      y_first_el = six.next(y)
      self._y = itertools.chain([y_first_el], y)
    else:
      y_first_el = None
      self._y = None
    self.n_classes = n_classes

    x_is_dict = isinstance(x_first_el, dict)
    y_is_dict = y is not None and isinstance(y_first_el, dict)
    if y_is_dict and n_classes is not None:
      assert isinstance(n_classes, dict)

    # extract shapes for first_elements
    if x_is_dict:
      x_first_el_shape = dict(
          [(k, [1] + list(v.shape)) for k, v in list(x_first_el.items())])
    else:
      x_first_el_shape = [1] + list(x_first_el.shape)

    if y_is_dict:
      y_first_el_shape = dict(
          [(k, [1] + list(v.shape)) for k, v in list(y_first_el.items())])
    elif y is None:
      y_first_el_shape = None
    else:
      y_first_el_shape = ([1] + list(y_first_el[0].shape if isinstance(
          y_first_el, list) else y_first_el.shape))

    self.input_shape, self.output_shape, self._batch_size = _get_in_out_shape(
        x_first_el_shape, y_first_el_shape, n_classes, batch_size)

    # Input dtype of x_first_el.
    if x_is_dict:
      self._input_dtype = dict(
          [(k, _check_dtype(v.dtype)) for k, v in list(x_first_el.items())])
    else:
      self._input_dtype = _check_dtype(x_first_el.dtype)

    # Output dtype of y_first_el.
    def check_y_dtype(el):
      if isinstance(el, np.ndarray):
        return el.dtype
      elif isinstance(el, list):
        return check_y_dtype(el[0])
      else:
        return _check_dtype(np.dtype(type(el)))

    # Output types are floats, due to both softmaxes and regression req.
    if n_classes is not None and (y is None or not y_is_dict) and n_classes > 0:
      self._output_dtype = np.float32
    elif y_is_dict:
      self._output_dtype = dict(
          [(k, check_y_dtype(v)) for k, v in list(y_first_el.items())])
    elif y is None:
      self._output_dtype = None
    else:
      self._output_dtype = check_y_dtype(y_first_el)

  def get_feed_params(self):
    """Function returns a `dict` with data feed params while training.

    Returns:
      A `dict` with data feed params while training.
    """
    return {'batch_size': self._batch_size}

  def get_feed_dict_fn(self):
    """Returns a function, that will sample data and provide it to placeholders.

    Returns:
      A function that when called samples a random subset of batch size
      from x and y.
    """
    self.stopped = False

    def _feed_dict_fn():
      """Samples data and provides it to placeholders.

      Returns:
        `dict` of input and output tensors.
      """

      def init_array(shape, dtype):
        """Initialize array of given shape or dict of shapes and dtype."""
        if shape is None:
          return None
        elif isinstance(shape, dict):
          return dict([(k, np.zeros(shape[k], dtype[k]))
                       for k in list(shape.keys())])
        else:
          return np.zeros(shape, dtype=dtype)

      def put_data_array(dest, index, source=None, n_classes=None):
        """Puts data array into container."""
        if source is None:
          dest = dest[:index]
        elif n_classes is not None and n_classes > 1:
          if len(self.output_shape) == 2:
            dest.itemset((index, source), 1.0)
          else:
            for idx, value in enumerate(source):
              dest.itemset(tuple([index, idx, value]), 1.0)
        else:
          if len(dest.shape) > 1:
            dest[index, :] = source
          else:
            dest[index] = source[0] if isinstance(source, list) else source
        return dest

      def put_data_array_or_dict(holder, index, data=None, n_classes=None):
        """Puts data array or data dictionary into container."""
        if holder is None:
          return None
        if isinstance(holder, dict):
          if data is None:
            data = {k: None for k in holder.keys()}
          assert isinstance(data, dict)
          for k in holder.keys():
            num_classes = n_classes[k] if (n_classes is not None and
                                           k in n_classes) else None
            holder[k] = put_data_array(holder[k], index, data[k], num_classes)
        else:
          holder = put_data_array(holder, index, data, n_classes)
        return holder

      if self.stopped:
        raise StopIteration

      inp = init_array(self.input_shape, self._input_dtype)
      out = init_array(self.output_shape, self._output_dtype)

      for i in xrange(self._batch_size):
        # Add handling when queue ends.
        try:
          next_inp = six.next(self._x)
          inp = put_data_array_or_dict(inp, i, next_inp, None)
        except StopIteration:
          self.stopped = True
          if i == 0:
            raise
          inp = put_data_array_or_dict(inp, i, None, None)
          out = put_data_array_or_dict(out, i, None, None)
          break

        if self._y is not None:
          next_out = six.next(self._y)
          out = put_data_array_or_dict(out, i, next_out, self.n_classes)

      # creating feed_dict
      if isinstance(inp, dict):
        feed_dict = dict([(self._input_placeholder[k].name, inp[k])
                          for k in list(self._input_placeholder.keys())])
      else:
        feed_dict = {self._input_placeholder.name: inp}
      if self._y is not None:
        if isinstance(out, dict):
          feed_dict.update(
              dict([(self._output_placeholder[k].name, out[k])
                    for k in list(self._output_placeholder.keys())]))
        else:
          feed_dict.update({self._output_placeholder.name: out})

      return feed_dict

    return _feed_dict_fn

class DaskDataFeeder(object):
  """Data feeder for that reads data from dask.Series and dask.DataFrame.

  Numpy arrays can be serialized to disk and it's possible to do random seeks
  into them. DaskDataFeeder will remove requirement to have full dataset in the
  memory and still do random seeks for sampling of batches.
  """

  def __init__(self,
               x,
               y,
               n_classes,
               batch_size,
               shuffle=True,
               random_state=None,
               epochs=None):
    """Initializes a DaskDataFeeder instance.

    Args:
      x: iterator that returns for each element, returns features.
      y: iterator that returns for each element, returns 1 or many classes /
        regression values.
      n_classes: indicator of how many classes the label has.
      batch_size: Mini batch size to accumulate.
      shuffle: Whether to shuffle the inputs.
      random_state: random state for RNG. Note that it will mutate so use a
        int value for this if you want consistent sized batches.
      epochs: Number of epochs to run.

    Attributes:
      x: input features.
      y: input label.
      n_classes: number of classes.
      batch_size: mini batch size to accumulate.
      input_shape: shape of the input.
      output_shape: shape of the output.
      input_dtype: dtype of input.
      output_dtype: dtype of output.

    Raises:
      ValueError: if `x` or `y` are `dict`, as they are not supported currently.
    """

    if isinstance(x, dict) or isinstance(y, dict):
      raise ValueError(
          'DaskDataFeeder does not support dictionaries at the moment.')

    # pylint: disable=invalid-name,super-init-not-called
    import dask.dataframe as dd  # pylint: disable=g-import-not-at-top
    # TODO(terrytangyuan): check x and y dtypes in dask_io like pandas
    self._x = x
    self._y = y
    # save column names
    self._x_columns = list(x.columns)
    if isinstance(y.columns[0], str):
      self._y_columns = list(y.columns)
    else:
      # deal with cases where two DFs have overlapped default numeric colnames
      self._y_columns = len(self._x_columns) + 1
      self._y = self._y.rename(columns={y.columns[0]: self._y_columns})

    # TODO(terrytangyuan): deal with unsupervised cases
    # combine into a data frame
    self.df = dd.multi.concat([self._x, self._y], axis=1)
    self.n_classes = n_classes

    x_count = x.count().compute()[0]
    x_shape = (x_count, len(self._x.columns))
    y_shape = (x_count, len(self._y.columns))
    # TODO(terrytangyuan): Add support for shuffle and epochs.
    self._shuffle = shuffle
    self.epochs = epochs
    self.input_shape, self.output_shape, self._batch_size = _get_in_out_shape(
        x_shape, y_shape, n_classes, batch_size)
    self.sample_fraction = self._batch_size / float(x_count)
    self._input_dtype = _check_dtype(self._x.dtypes[0])
    self._output_dtype = _check_dtype(self._y.dtypes[self._y_columns])
    if random_state is None:
      self.random_state = 66
    else:
      self.random_state = random_state

  def get_feed_params(self):
    """Function returns a `dict` with data feed params while training.

    Returns:
      A `dict` with data feed params while training.
    """
    return {'batch_size': self._batch_size}

  def get_feed_dict_fn(self, input_placeholder, output_placeholder):
    """Returns a function, that will sample data and provide it to placeholders.

    Args:
      input_placeholder: tf.Placeholder for input features mini batch.
      output_placeholder: tf.Placeholder for output labels.

    Returns:
      A function that when called samples a random subset of batch size
      from x and y.
    """

    def _feed_dict_fn():
      """Samples data and provides it to placeholders."""
      # TODO(ipolosukhin): option for with/without replacement (dev version of
      # dask)
      sample = self.df.random_split(
          [self.sample_fraction, 1 - self.sample_fraction],
          random_state=self.random_state)
      inp = extract_pandas_matrix(sample[0][self._x_columns].compute()).tolist()
      out = extract_pandas_matrix(sample[0][self._y_columns].compute())
      # convert to correct dtype
      inp = np.array(inp, dtype=self._input_dtype)
      # one-hot encode out for each class for cross entropy loss
      if HAS_PANDAS:
        import pandas as pd  # pylint: disable=g-import-not-at-top
        if not isinstance(out, pd.Series):
          out = out.flatten()
      out_max = self._y.max().compute().values[0]
      encoded_out = np.zeros((out.size, out_max + 1), dtype=self._output_dtype)
      encoded_out[np.arange(out.size), out] = 1
      return {input_placeholder.name: inp, output_placeholder.name: encoded_out}

    return _feed_dict_fn


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import six

import warnings

import matplotlib.axes as maxes
from matplotlib.artist import Artist
from matplotlib.axis import XAxis, YAxis

class SimpleChainedObjects(object):
    def __init__(self, objects):
        self._objects = objects

    def __getattr__(self, k):
        _a = SimpleChainedObjects([getattr(a, k) for a in self._objects])
        return _a

    def __call__(self, *kl, **kwargs):
        for m in self._objects:
            m(*kl, **kwargs)

class Axes(maxes.Axes):
    def toggle_axisline(self, b):
        warnings.warn("toggle_axisline is not necessary and deprecated in axes_grid1")

    class AxisDict(dict):
        def __init__(self, axes):
            self.axes = axes
            super(Axes.AxisDict, self).__init__()

        def __getitem__(self, k):
            if isinstance(k, tuple):
                r = SimpleChainedObjects([dict.__getitem__(self, k1) for k1 in k])
                return r
            elif isinstance(k, slice):
                if k.start == None and k.stop == None and k.step == None:
                    r = SimpleChainedObjects(list(six.itervalues(self)))
                    return r
                else:
                    raise ValueError("Unsupported slice")
            else:
                return dict.__getitem__(self, k)

        def __call__(self, *v, **kwargs):
            return maxes.Axes.axis(self.axes, *v, **kwargs)

    def __init__(self, *kl, **kw):
        super(Axes, self).__init__(*kl, **kw)

    def _init_axis_artists(self, axes=None):
        if axes is None:
            axes = self

        self._axislines = self.AxisDict(self)

        self._axislines["bottom"] = SimpleAxisArtist(self.xaxis, 1, self.spines["bottom"])
        self._axislines["top"] = SimpleAxisArtist(self.xaxis, 2, self.spines["top"])
        self._axislines["left"] = SimpleAxisArtist(self.yaxis, 1, self.spines["left"])
        self._axislines["right"] = SimpleAxisArtist(self.yaxis, 2, self.spines["right"])

    def _get_axislines(self):
        return self._axislines

    axis = property(_get_axislines)

    def cla(self):

        super(Axes, self).cla()
        self._init_axis_artists()

class SimpleAxisArtist(Artist):
    def __init__(self, axis, axisnum, spine):
        self._axis = axis
        self._axisnum = axisnum
        self.line = spine

        if isinstance(axis, XAxis):
            self._axis_direction = ["bottom", "top"][axisnum-1]
        elif isinstance(axis, YAxis):
            self._axis_direction = ["left", "right"][axisnum-1]
        else:
            raise ValueError("axis must be instance of XAxis or YAxis : %s is provided" % (axis,))
        Artist.__init__(self)

    def _get_major_ticks(self):
        tickline = "tick%dline" % self._axisnum
        return SimpleChainedObjects([getattr(tick, tickline) for tick \
                                     in self._axis.get_major_ticks()])

    def _get_major_ticklabels(self):
        label = "label%d" % self._axisnum
        return SimpleChainedObjects([getattr(tick, label) for tick \
                                     in self._axis.get_major_ticks()])

    def _get_label(self):
        return self._axis.label

    major_ticks = property(_get_major_ticks)
    major_ticklabels = property(_get_major_ticklabels)
    label = property(_get_label)

    def set_visible(self, b):
        self.toggle(all=b)
        self.line.set_visible(b)
        self._axis.set_visible(True)
        Artist.set_visible(self, b)

    def set_label(self, txt):
        self._axis.set_label_text(txt)

    def toggle(self, all=None, ticks=None, ticklabels=None, label=None):

        if all:
            _ticks, _ticklabels, _label = True, True, True
        elif all is not None:
            _ticks, _ticklabels, _label = False, False, False
        else:
            _ticks, _ticklabels, _label = None, None, None

        if ticks is not None:
            _ticks = ticks
        if ticklabels is not None:
            _ticklabels = ticklabels
        if label is not None:
            _label = label

        tickOn = "tick%dOn" % self._axisnum
        labelOn = "label%dOn" % self._axisnum

        if _ticks is not None:
            tickparam = {tickOn: _ticks}
            self._axis.set_tick_params(**tickparam)
        if _ticklabels is not None:
            tickparam = {labelOn: _ticklabels}
            self._axis.set_tick_params(**tickparam)

        if _label is not None:
            pos = self._axis.get_label_position()
            if (pos == self._axis_direction) and not _label:
                self._axis.label.set_visible(False)
            elif _label:
                self._axis.label.set_visible(True)
                self._axis.set_label_position(self._axis_direction)

if __name__ == '__main__':
    fig = figure()
    ax = Axes(fig, [0.1, 0.1, 0.8, 0.8])
    fig.add_axes(ax)
    ax.cla()


#
# Copyright 2013 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import deque

import pytz
import numpy as np
import pandas as pd

from datetime import datetime
from unittest import TestCase

from zipline.utils.test_utils import setup_logger

from zipline.sources.data_source import DataSource
import zipline.utils.factory as factory

from zipline.transforms import batch_transform

from zipline.test_algorithms import (BatchTransformAlgorithm,
                                     BatchTransformAlgorithmMinute,
                                     ReturnPriceBatchTransform)

from zipline.algorithm import TradingAlgorithm
from zipline.utils.tradingcalendar import trading_days
from copy import deepcopy

@batch_transform
def return_price(data):
    return data.price

class BatchTransformAlgorithmSetSid(TradingAlgorithm):
    def initialize(self, sids=None):
        self.history = []

        self.batch_transform = return_price(
            refresh_period=1,
            window_length=10,
            clean_nans=False,
            sids=sids,
            compute_only_full=False
        )

    def handle_data(self, data):
        self.history.append(
            deepcopy(self.batch_transform.handle_data(data)))

class DifferentSidSource(DataSource):
    def __init__(self):
        self.dates = pd.date_range('1990-01-01', periods=180, tz='utc')
        self.start = self.dates[0]
        self.end = self.dates[-1]
        self._raw_data = None
        self.sids = range(90)
        self.sid = 0
        self.trading_days = []

    @property
    def instance_hash(self):
        return '1234'

    @property
    def raw_data(self):
        if not self._raw_data:
            self._raw_data = self.raw_data_gen()
        return self._raw_data

    @property
    def mapping(self):
        return {
            'dt': (lambda x: x, 'dt'),
            'sid': (lambda x: x, 'sid'),
            'price': (float, 'price'),
            'volume': (int, 'volume'),
        }

    def raw_data_gen(self):
        # Create differente sid for each event
        for date in self.dates:
            if date not in trading_days:
                continue
            event = {'dt': date,
                     'sid': self.sid,
                     'price': self.sid,
                     'volume': self.sid}
            self.sid += 1
            self.trading_days.append(date)
            yield event

class TestChangeOfSids(TestCase):
    def setUp(self):
        self.sids = range(90)
        self.sim_params = factory.create_simulation_parameters(
            start=datetime(1990, 1, 1, tzinfo=pytz.utc),
            end=datetime(1990, 1, 8, tzinfo=pytz.utc)
        )

    def test_all_sids_passed(self):
        algo = BatchTransformAlgorithmSetSid(sim_params=self.sim_params)
        source = DifferentSidSource()
        algo.run(source)
        for i, (df, date) in enumerate(zip(algo.history, source.trading_days)):
            self.assertEqual(df.index[-1], date, "Newest event doesn't \
                             match.")

            for sid in self.sids[:i]:
                self.assertIn(sid, df.columns)

            last_elem = len(df) - 1
            self.assertEqual(df[last_elem][last_elem], last_elem)

class TestBatchTransformMinutely(TestCase):
    def setUp(self):
        start = pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc)
        end = pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc)
        self.sim_params = factory.create_simulation_parameters(
            start=start,
            end=end,
        )
        self.sim_params.emission_rate = 'daily'
        self.sim_params.data_frequency = 'minute'
        setup_logger(self)
        self.source, self.df = \
            factory.create_test_df_source(bars='minute')

    def test_core(self):
        algo = BatchTransformAlgorithmMinute(sim_params=self.sim_params)
        algo.run(self.source)
        wl = int(algo.window_length * 6.5 * 60)
        for bt in algo.history[wl:]:
            self.assertEqual(len(bt), wl)

    def test_window_length(self):
        algo = BatchTransformAlgorithmMinute(sim_params=self.sim_params,
                                             window_length=1, refresh_period=0)
        algo.run(self.source)
        wl = int(algo.window_length * 6.5 * 60)
        np.testing.assert_array_equal(algo.history[:(wl - 1)],
                                      [None] * (wl - 1))
        for bt in algo.history[wl:]:
            self.assertEqual(len(bt), wl)

class TestBatchTransform(TestCase):
    def setUp(self):
        self.sim_params = factory.create_simulation_parameters(
            start=datetime(1990, 1, 1, tzinfo=pytz.utc),
            end=datetime(1990, 1, 8, tzinfo=pytz.utc)
        )
        setup_logger(self)
        self.source, self.df = \
            factory.create_test_df_source(self.sim_params)

    def test_core_functionality(self):
        algo = BatchTransformAlgorithm(sim_params=self.sim_params)
        algo.run(self.source)
        wl = algo.window_length
        # The following assertion depend on window length of 3
        self.assertEqual(wl, 3)
        # If window_length is 3, there should be 2 None events, as the
        # window fills up on the 3rd day.
        n_none_events = 2
        self.assertEqual(algo.history_return_price_class[:n_none_events],
                         [None] * n_none_events,
                         "First two iterations should return None." + "\n" +
                         "i.e. no returned values until window is full'" +
                         "%s" % (algo.history_return_price_class,))
        self.assertEqual(algo.history_return_price_decorator[:n_none_events],
                         [None] * n_none_events,
                         "First two iterations should return None." + "\n" +
                         "i.e. no returned values until window is full'" +
                         "%s" % (algo.history_return_price_decorator,))

        # After three Nones, the next value should be a data frame
        self.assertTrue(isinstance(
            algo.history_return_price_class[wl],
            pd.DataFrame)
        )

        # Test whether arbitrary fields can be added to datapanel
        field = algo.history_return_arbitrary_fields[-1]
        self.assertTrue(
            'arbitrary' in field.items,
            'datapanel should contain column arbitrary'
        )

        self.assertTrue(all(
            field['arbitrary'].values.flatten() ==
            [123] * algo.window_length),
            'arbitrary dataframe should contain only "test"'
        )

        for data in algo.history_return_sid_filter[wl:]:
            self.assertIn(0, data.columns)
            self.assertNotIn(1, data.columns)

        for data in algo.history_return_field_filter[wl:]:
            self.assertIn('price', data.items)
            self.assertNotIn('ignore', data.items)

        for data in algo.history_return_field_no_filter[wl:]:
            self.assertIn('price', data.items)
            self.assertIn('ignore', data.items)

        for data in algo.history_return_ticks[wl:]:
            self.assertTrue(isinstance(data, deque))

        for data in algo.history_return_not_full:
            self.assertIsNot(data, None)

        # test overloaded class
        for test_history in [algo.history_return_price_class,
                             algo.history_return_price_decorator]:
            # starting at window length, the window should contain
            # consecutive (of window length) numbers up till the end.
            for i in range(algo.window_length, len(test_history)):
                np.testing.assert_array_equal(
                    range(i - algo.window_length + 2, i + 2),
                    test_history[i].values.flatten()
                )

    def test_passing_of_args(self):
        algo = BatchTransformAlgorithm(1, kwarg='str',
                                       sim_params=self.sim_params)
        self.assertEqual(algo.args, (1,))
        self.assertEqual(algo.kwargs, {'kwarg': 'str'})

        algo.run(self.source)
        expected_item = ((1, ), {'kwarg': 'str'})
        self.assertEqual(
            algo.history_return_args,
            [
                # 1990-01-01 - market holiday, no event
                # 1990-01-02 - window not full
                None,
                # 1990-01-03 - window not full
                None,
                # 1990-01-04 - window now full, 3rd event
                expected_item,
                # 1990-01-05 - window now full
                expected_item,
                # 1990-01-08 - window now full
                expected_item
            ])

def run_batchtransform(window_length=10):
    sim_params = factory.create_simulation_parameters(
        start=datetime(1990, 1, 1, tzinfo=pytz.utc),
        end=datetime(1995, 1, 8, tzinfo=pytz.utc)
    )
    source, df = factory.create_test_df_source(sim_params)

    return_price_class = ReturnPriceBatchTransform(
        refresh_period=1,
        window_length=window_length,
        clean_nans=False
    )

    for raw_event in source:
        raw_event['datetime'] = raw_event.dt
        event = {0: raw_event}
        return_price_class.handle_data(event)


"""Test truncated SVD transformer."""

import numpy as np
import scipy.sparse as sp

from sklearn.decomposition import TruncatedSVD
from sklearn.utils import check_random_state
from sklearn.utils.testing import (assert_array_almost_equal, assert_equal,
                                   assert_raises, assert_greater,
                                   assert_array_less)

# Make an X that looks somewhat like a small tf-idf matrix.
# XXX newer versions of SciPy have scipy.sparse.rand for this.
shape = 60, 55
n_samples, n_features = shape
rng = check_random_state(42)
X = rng.randint(-100, 20, np.product(shape)).reshape(shape)
X = sp.csr_matrix(np.maximum(X, 0), dtype=np.float64)
X.data[:] = 1 + np.log(X.data)
Xdense = X.A

def test_algorithms():
    svd_a = TruncatedSVD(30, algorithm="arpack")
    svd_r = TruncatedSVD(30, algorithm="randomized", random_state=42)

    Xa = svd_a.fit_transform(X)[:, :6]
    Xr = svd_r.fit_transform(X)[:, :6]
    assert_array_almost_equal(Xa, Xr)

    comp_a = np.abs(svd_a.components_)
    comp_r = np.abs(svd_r.components_)
    # All elements are equal, but some elements are more equal than others.
    assert_array_almost_equal(comp_a[:9], comp_r[:9])
    assert_array_almost_equal(comp_a[9:], comp_r[9:], decimal=3)

def test_attributes():
    for n_components in (10, 25, 41):
        tsvd = TruncatedSVD(n_components).fit(X)
        assert_equal(tsvd.n_components, n_components)
        assert_equal(tsvd.components_.shape, (n_components, n_features))

def test_too_many_components():
    for algorithm in ["arpack", "randomized"]:
        for n_components in (n_features, n_features+1):
            tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm)
            assert_raises(ValueError, tsvd.fit, X)

def test_sparse_formats():
    for fmt in ("array", "csr", "csc", "coo", "lil"):
        Xfmt = Xdense if fmt == "dense" else getattr(X, "to" + fmt)()
        tsvd = TruncatedSVD(n_components=11)
        Xtrans = tsvd.fit_transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))
        Xtrans = tsvd.transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))

def test_inverse_transform():
    for algo in ("arpack", "randomized"):
        # We need a lot of components for the reconstruction to be "almost
        # equal" in all positions. XXX Test means or sums instead?
        tsvd = TruncatedSVD(n_components=52, random_state=42)
        Xt = tsvd.fit_transform(X)
        Xinv = tsvd.inverse_transform(Xt)
        assert_array_almost_equal(Xinv, Xdense, decimal=1)

def test_integers():
    Xint = X.astype(np.int64)
    tsvd = TruncatedSVD(n_components=6)
    Xtrans = tsvd.fit_transform(Xint)
    assert_equal(Xtrans.shape, (n_samples, tsvd.n_components))

def test_explained_variance():
    # Test sparse data
    svd_a_10_sp = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_sp = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_sp = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_sp = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_sp = svd_a_10_sp.fit_transform(X)
    X_trans_r_10_sp = svd_r_10_sp.fit_transform(X)
    X_trans_a_20_sp = svd_a_20_sp.fit_transform(X)
    X_trans_r_20_sp = svd_r_20_sp.fit_transform(X)

    # Test dense data
    svd_a_10_de = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_de = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_de = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_de = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_de = svd_a_10_de.fit_transform(X.toarray())
    X_trans_r_10_de = svd_r_10_de.fit_transform(X.toarray())
    X_trans_a_20_de = svd_a_20_de.fit_transform(X.toarray())
    X_trans_r_20_de = svd_r_20_de.fit_transform(X.toarray())

    # helper arrays for tests below
    svds = (svd_a_10_sp, svd_r_10_sp, svd_a_20_sp, svd_r_20_sp, svd_a_10_de,
            svd_r_10_de, svd_a_20_de, svd_r_20_de)
    svds_trans = (
        (svd_a_10_sp, X_trans_a_10_sp),
        (svd_r_10_sp, X_trans_r_10_sp),
        (svd_a_20_sp, X_trans_a_20_sp),
        (svd_r_20_sp, X_trans_r_20_sp),
        (svd_a_10_de, X_trans_a_10_de),
        (svd_r_10_de, X_trans_r_10_de),
        (svd_a_20_de, X_trans_a_20_de),
        (svd_r_20_de, X_trans_r_20_de),
    )
    svds_10_v_20 = (
        (svd_a_10_sp, svd_a_20_sp),
        (svd_r_10_sp, svd_r_20_sp),
        (svd_a_10_de, svd_a_20_de),
        (svd_r_10_de, svd_r_20_de),
    )
    svds_sparse_v_dense = (
        (svd_a_10_sp, svd_a_10_de),
        (svd_a_20_sp, svd_a_20_de),
        (svd_r_10_sp, svd_r_10_de),
        (svd_r_20_sp, svd_r_20_de),
    )

    # Assert the 1st component is equal
    for svd_10, svd_20 in svds_10_v_20:
        assert_array_almost_equal(
            svd_10.explained_variance_ratio_,
            svd_20.explained_variance_ratio_[:10],
            decimal=5,
        )

    # Assert that 20 components has higher explained variance than 10
    for svd_10, svd_20 in svds_10_v_20:
        assert_greater(
            svd_20.explained_variance_ratio_.sum(),
            svd_10.explained_variance_ratio_.sum(),
        )

    # Assert that all the values are greater than 0
    for svd in svds:
        assert_array_less(0.0, svd.explained_variance_ratio_)

    # Assert that total explained variance is less than 1
    for svd in svds:
        assert_array_less(svd.explained_variance_ratio_.sum(), 1.0)

    # Compare sparse vs. dense
    for svd_sparse, svd_dense in svds_sparse_v_dense:
        assert_array_almost_equal(svd_sparse.explained_variance_ratio_,
                svd_dense.explained_variance_ratio_)

    # Test that explained_variance is correct
    for svd, transformed in svds_trans:
        total_variance = np.var(X.toarray(), axis=0).sum()
        variances = np.var(transformed, axis=0)
        true_explained_variance_ratio = variances / total_variance

        assert_array_almost_equal(
            svd.explained_variance_ratio_,
            true_explained_variance_ratio,
        )


#!/usr/bin/python2.7
# encoding: utf-8

from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.tri as Tri
import matplotlib.ticker as ticker
import seaborn

class PlotsTidegauge:
    """'Plots' subset of Tidegauge class gathers plotting functions"""
    def __init__(self, variable, debug=False):
        self._var = variable

    def plot_xy(self, x, y, title=' ', xLabel=' ', yLabel=' '):
        """
        Simple X vs Y plot

        Inputs:
        ------
          - x = 1D array
          - y = 1D array
        """
        fig = plt.figure(figsize=(18,10))
        plt.rc('font',size='22')
        self._fig = plt.plot(x, y, label=title)
        scale = 1
        ticks = ticker.FuncFormatter(lambda lon, pos: '{0:g}'.format(lon/scale))
        plt.ylabel(yLabel)
        plt.xlabel(xLabel)
        #plt.legend()
        plt.show()

#TR_comments: templates
#    def whatever(self, debug=False):
#        if debug or self._debug:
#            print 'Start whatever...'
#
#        if debug or self._debug:
#            print '...Passed'


#!/usr/bin/env python2.7
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
from tika import parser
import pandas as pd
from vector import Vector
from sklearn.cluster import KMeans
import argparse, os, json

def filterFiles(inputDir, acceptTypes):
    filename_list = []

    for root, dirnames, files in os.walk(inputDir):
        dirnames[:] = [d for d in dirnames if not d.startswith('.')]
        for filename in files:
            if not filename.startswith('.'):
                filename_list.append(os.path.join(root, filename))

    filename_list = (filename for filename in filename_list if "metadata" in parser.from_file(filename))
    if acceptTypes:
        filename_list = (filename for filename in filename_list if str(parser.from_file(filename)['metadata']['Content-Type'].encode('utf-8').decode('utf-8')).split('/')[-1] in acceptTypes)
    else:
        print("Accepting all MIME Types.....")

    return filename_list

if __name__ == "__main__":

    argParser = argparse.ArgumentParser('k-means Clustering of documents based on metadata values')
    argParser.add_argument('--inputDir', required=True, help='path to directory containing files')
    argParser.add_argument('--outJSON', required=True, help='/path/to/clusters.json containing k-means cluster assignments')
    argParser.add_argument('--Kvalue', help='number of clusters to find')
    #argParser.add_argument('--findK', action='store_true', help='find the optimal value of K')
    argParser.add_argument('--accept', nargs='+', type=str, help='Optional: compute similarity only on specified IANA MIME Type(s)')
    args = argParser.parse_args()

    # cluster for a particular value of K
    # if args.inputDir and args.outJSON and args.findK:

    if args.inputDir and args.outJSON and args.Kvalue:

        list_of_points = []
        for eachFile in filterFiles(args.inputDir, args.accept):
            list_of_points.append(Vector(eachFile, parser.from_file(eachFile)["metadata"]))

        list_of_Dicts = (point.features for point in list_of_points)

        df = pd.DataFrame(list_of_Dicts)
        df = df.fillna(0)

        print(df.shape)

        kmeans = KMeans(n_clusters=int(args.Kvalue),
                        init='k-means++',
                        max_iter=300,   # k-means convergence
                        n_init=10,      # find global minima
                        n_jobs=-2,    # parallelize
                        )

        labels = kmeans.fit_predict(df)  # unsupervised (X, y=None)

        print(labels)    # kmeans.labels_

        clusters = {}
        for i in range(0, len(labels)):

            node = { "metadata": json.dumps(list_of_points[i].features),
                     "name": list_of_points[i].filename.split('/')[-1],
                     "path": list_of_points[i].filename
                     }
            try:
                clusters[str(labels[i])].append(node)
            except KeyError:
                clusters[str(labels[i])] = []
                clusters[str(labels[i])].append(node)

        # generate clusters.JSON
        with open(args.outJSON, "w") as jsonF:

            json_data = {"name": "clusters"}
            children = []

            for key in clusters:
                cluster_children = {"name": "cluster"+key, "children": clusters[key]}
                children.append(cluster_children)

            json_data["children"] = children
            json.dump(json_data, jsonF)

        # print matplotlib
        # user chooses k => generates k

        # find elbow

        #kmeans.transform()

        # String Length Of Course
        # df.to_csv("bashhshs.csv", sep=',')