# simple trading strategy simulator

import pandas as pd
from pandas.tools.plotting import autocorrelation_plot
from pandas.tools.plotting import scatter_matrix

import numpy as np
from scipy import stats

import sklearn
from sklearn import preprocessing as pp

import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib import interactive
interactive(True)

import sys
import time

import logging as log
log.basicConfig(level=log.DEBUG)

import glob
import os.path
import pickle

import logging as log
log.basicConfig(level=log.DEBUG)

import random
import pdb

pd.set_option('display.width',500)

# define constant friction function
DefaultBPS = 10
def FrictionInBps(U, cfg, kvargs):
    """ default FrictionInBps function just returns default,
        but the interface receives all strategy info after
        strategy is run, so one can create more realistic
        impact models """
    return DefaultBPS

""" default simulator cfg dictionary.
    default keys/values:
    FrictionInBps - function that takes same args as strategy.
                    by default, returns DefaultBps.
    InitBal - in $s
    Reinvest - should we reinvest our winnings or constantly assume we have InitBal?
    Verbose
"""
DEF_SIM_CFG= { 'FrictionInBps': FrictionInBps,
               'Verbose'      : True,
               'InitBal'      : 1e7,
               'Reinvest'     : True  }

# columns in prepped univ
SIM_COLS = ["Sym","Product","Instrument",
            "Multiplier","Expiry","Strike",
            "Open","High","Low","Close","Volume"]

SIM_COLS_OUT = ["Prev_Weight", "Weight", "Prev_Qty", "Qty",
                "Trade_Qty",  "Trade_Fric", "PNL", "NET_PNL"]

SIM_COL_BALS =[ "NAV","Friction","PNL","NET_PNL", "Longs","Shorts",
                "Long_Dlrs","Short_Dlrs","Num_Trades","Turnover","NET_Return"]

def squarem( df, sym='Sym', min_pct=.9 ) :
# sim_squarem solves the common problem in which you have a large table of
#  data grouped by symbols, some of which have missing data.  You want to
#  'square' the data such that any symbol which is missing 'too much' data
#  is expunged and the remaining data is filled appropriately, leaving you
#  with a dataset which has the same # of observations for each symbol.
#
    bysyms = df.groupby(sym).size()
    idx = df.index.unique()
    onumsyms = len(bysyms)

    minlen = int(round(len(idx) * .9 ))
    keep = bysyms[bysyms > minlen]
    u = df[ df[sym].isin(keep.index) ]
    numsyms = len(keep)
    log.info('Got rid of %d/%d symbols',(numsyms-onumsyms),onumsyms)

    u.replace(0,np.nan,inplace=True)
    u.replace([np.inf, -np.inf], np.nan,inplace=True)

    u.sort_index(inplace=True)

    uidx = u.index.unique()
    # groupby and reindex magic
    z = u.groupby(sym).apply(
        lambda x:  x.reindex(uidx).ffill()).reset_index(0,drop=True)

    # badz = z[z.isnull().any(axis=1)]
    # if len(badz.index) > 0 :
    #     badtimes = badz.index.unique().values
    #     z.drop( badtimes, inplace=True )
    #     for dt in badtimes:
    #         log.info('removed %s for NaNs',pd.to_datetime(str(dt)).strftime(
    #             '%Y-%m-%d'))
    return z

def prep_univ( dateTime, symbol,
               open, high, low, close, volume,
               product, instrument='STK', multiplier=1.0,expiry=None,
               strike=None,adv_days=20,sd_days=20, open2close_returns=True,
               scaleAndCenter=False,  **more_cols) :
    # constructs universe appropriate for use with simulator; any additional columns
    #  passed-in via ellipsis will be added to table as named
    #

    U = pd.DataFrame({'Sym': symbol,
                      'Product' : product, 'Instrument':instrument,
                      'Multiplier': 1.0, 'Expiry': None, 'Strike':None,
                      'Open':open,'High':high, 'Low':low, 'Close':close,
                      'Volume':volume }, index=dateTime )
    U = U[ SIM_COLS ]
    if len(more_cols) > 0:
        U = pd.concat( [U, pd.DataFrame(more_cols)], axis=1 )

    U.reset_index( inplace=True)
    U.sort_values(['Sym','Date'],inplace=True)
    U.Date = pd.to_datetime(U.Date)
    U.set_index('Date',inplace=True)

    if scaleAndCenter :
        log.debug('prep_univ: scaling & centering')
        raw_scaled = U.groupby('Sym').transform(
            lambda x : (x - x.mean())/x.std())
        U = pd.concat([ u.Sym, raw_scaled], axis=1)

    # calculate adv, returns, fwd_returns & change in volume
    U['ADV'] = U.groupby('Sym')['Volume'].apply(
        pd.rolling_mean, adv_days, 1).shift()
    U['DeltaV'] = U.groupby('Sym')['Volume'].transform(
        lambda x : np.log(x / x.shift()) )
    U['Return'] = U.groupby('Sym')['Close'].transform(
        lambda x : np.log(x / x.shift()) )
    U['Fwd_Close'] = U.groupby('Sym')['Close'].shift(-1)
    U['Fwd_Return'] = U.groupby('Sym')['Close'].transform(
        lambda x : np.log(x / x.shift()).shift(-1) ) # fwd.returns
    U['SD'] = U.groupby('Sym')['Return'].apply(
        pd.rolling_std, sd_days, 1).shift()

    if open2close_returns:
        U['Fwd_Open'] = U.groupby('Sym')['Open'].shift(-1)
        U['Fwd_COReturn'] = np.divide(np.add( U.Fwd_Open, -U.Close ),U.Close)

    U.ffill(inplace=True)
    U.sort_index(inplace=True)

    return U

# simple, default strategy: equal weight universe on daily basis
def eq_wt( U, cfg, kvargs ) :
    #pdb.set_trace()
    U.Weight = 1/float(len(U.index))
    return U

# given today's Universe U and Yesterday's Y, set U's
#  Prev_Weight and Prev_Qty to Y's Weight & Qty
#  TODO: clean-up
def _getprevs( U, Y ) :
    #  TODO: surely there's a cleaner way to do this...
    wts = Y.reset_index()[['Sym','Weight']]
    wts.columns = ['Sym','Prev_Weight']
    pwts = U[['Sym']].merge( wts, on = 'Sym' )['Prev_Weight']
    U.Prev_Weight=pwts.values
    qts = Y.reset_index()[['Sym','Qty']]
    qts.columns = ['Sym','Prev_Qty']
    pqts = U[['Sym']].merge( qts, on = 'Sym' )['Prev_Qty']
    U.Prev_Qty=pqts.values

# functor to run strategy each day and update tbls ...
# TODO: clean-up
def __sim ( U, FUN, cfg, B, kvargs) :
    # run sim to set weights
    U = FUN( U, cfg, kvargs)

    # set prev values for weight & qty...
    Y = kvargs.pop('_Y', None)
    if Y is not None and not np.all(Y.index==U.index):
        _getprevs(U,Y)
        loop = 1 + int(kvargs.pop('_L'))
    else:
        loop = 0
    kvargs['_L'] = loop
    kvargs['_Y'] = U

    bb = B.iloc[loop]
    #  fill-out trade details
    NAV = bb.NAV
    tospend = NAV/U.Weight
    U.Qty = np.round((NAV*U.Weight) / (U.Multiplier*U.Close))
    U.Trade_Qty  = U.Qty - U.Prev_Qty
    fbps = 1e-4 * cfg['FrictionInBps'](U,cfg,kvargs)
    U.Trade_Fric = U.Trade_Qty * U.Close * U.Multiplier * fbps
    U.PNL = (U.Fwd_Close - U.Close) * U.Qty * U.Multiplier
    U.NET_PNL = U.PNL - U.Trade_Fric

    # today's balances are based on yesterday's posns...
    longs = U[U.Qty > 0]
    shorts = U[U.Qty < 0]
    trades = U[U.Trade_Qty != 0]

    bb.Friction = U.Trade_Fric.sum()
    bb.PNL = U.PNL.sum()
    bb.NET_PNL = U.NET_PNL.sum()
    bb.Longs = len(longs.index)
    bb.Shorts = len(shorts.index)
    bb.Long_Dlrs = (longs.Close * longs.Multiplier * longs.Qty).sum()
    bb.Short_Dlrs = (shorts.Close * shorts.Multiplier * shorts.Qty).sum()
    bb.Num_Trades = len(trades.index)
    bb.Turnover = (trades.Close * trades.Multiplier
                   * trades.Trade_Qty.abs()).sum()/NAV

    if loop > 0 :
        yb = B.iloc[loop-1]
        ynav = yb.NAV
        tnav = ynav + yb.NET_PNL
        bb.NAV = tnav
        bb.NET_Return = (tnav-ynav)/ynav

    B.iloc[loop] = bb

    # pdb.set_trace()
    return U

def sim( univ, sim_FUN=eq_wt, cfg=DEF_SIM_CFG.copy(), kvargs={} ) :
    """ simulator: runs simulation and returns a table of activity and balances.
        args:
          univ - historical data that's been produced by prep_univ
          sim_FUN - strategy function.  by default, equal weights univ.
          cfg - cfg info.  by default
          kvargs - strat-specific extra data in a dict

    """
#
    t0 = time.time()
    all_times = univ.index.unique().values

    # prepare writable/output side of universe
    W = pd.DataFrame( columns=SIM_COLS_OUT, index = univ.index).fillna(0.0)
    U = pd.concat( [univ, W], axis=1 )

    # create balances table: one per day
    B = pd.DataFrame( columns = SIM_COL_BALS, index = all_times ).fillna(0.0)
    B.NAV = cfg['InitBal']

    # 'daily' loop
    Z = U.groupby(U.index).apply( __sim, FUN=sim_FUN,
                                  cfg=cfg, B=B, kvargs=kvargs )

    log.info('ran over %d days and %d rows in %d secs', len(all_times),
             len(U.index),time.time()-t0)

    # summarize results a bit more...?
    #ts=xts(B$Net.Return,order.by=B$DateTime)

    # return universe and balances
    #list(U=U,B=B, ts=ts)
    return Z, B

def sharpe(Returns) :
    return np.sqrt(252) * np.mean(Returns)/np.std(Returns)

def random_strat( U, cfg, kvargs ) :
    # random portfolio strategy: picks 'num_names' randomly
    nnames = kvargs.get('num_names',10)
    names = random.sample(U.Sym, nnames )
    U.Weight = np.where( U.Sym.isin( names ), 1/float(nnames), 0 )

    return U

def best_strat( U, cfg, kvargs ) :
    # portfolio strategy: picks 'num_names' based on trailing return
    nnames = kvargs.get('num_names',10)
    #pdb.set_trace()
    best = U.sort_values('Return',ascending=False,
                         na_position='last')['Sym'].head(10).values
    U.Weight = np.where( U.Sym.isin( best ), 1/float(nnames), 0 )
    return U

def worst_strat( U, cfg, kvargs ) :
    # portfolio strategy: picks 'num_names' based on trailing return
    nnames = kvargs.get('num_names',10)
    #pdb.set_trace()
    worst = U.sort_values('Return',ascending=True,
                          na_position='last')['Sym'].head(10).values
    U.Weight = np.where( U.Sym.isin( worst ), 1/float(nnames), 0 )
    return U

def rtest(U,FUN=random_strat, runs=10):
# run given strat repeatedly, plotting NAVs and Returning them
#   nb: this only makes sense if the strategy is random...
    # run random_strat 'runs' times and plot NAVs
    N = None
    for i in range(runs) :
        _,b = sim( U, sim_FUN=FUN )
        n = pd.DataFrame(b.NAV)
        N = n if N is None else pd.concat([N,n],axis=1)
    N.plot(legend=False)
    return N

def sim_test():
# dev driver
    f = 'U.pkl'
    P = pickle.load(open(f))
    log.info('loaded <%s>',f)
    P.describe()
    U = P[P.index >= '2005-01-01']
    U.describe()
    import sim
    _,B = sim.sim(U)
    #plot NAV
    B.NAV.plot(title='Equal Weight Everyone')
    return B


# Copyright (c) 2010 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause
"""Glue code to load http://mlcomp.org data as a scikit.learn dataset"""

import os
import numbers
from sklearn.datasets.base import load_files

def _load_document_classification(dataset_path, metadata, set_=None, **kwargs):
    if set_ is not None:
        dataset_path = os.path.join(dataset_path, set_)
    return load_files(dataset_path, metadata.get('description'), **kwargs)

LOADERS = {
    'DocumentClassification': _load_document_classification,
    # TODO: implement the remaining domain formats
}

def load_mlcomp(name_or_id, set_="raw", mlcomp_root=None, **kwargs):
    """Load a datasets as downloaded from http://mlcomp.org

    Parameters
    ----------

    name_or_id : the integer id or the string name metadata of the MLComp
                 dataset to load

    set_ : select the portion to load: 'train', 'test' or 'raw'

    mlcomp_root : the filesystem path to the root folder where MLComp datasets
                  are stored, if mlcomp_root is None, the MLCOMP_DATASETS_HOME
                  environment variable is looked up instead.

    **kwargs : domain specific kwargs to be passed to the dataset loader.

    Read more in the :ref:`User Guide <datasets>`.

    Returns
    -------

    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'filenames', the files holding the raw to learn, 'target', the
        classification labels (integer index), 'target_names',
        the meaning of the labels, and 'DESCR', the full description of the
        dataset.

    Note on the lookup process: depending on the type of name_or_id,
    will choose between integer id lookup or metadata name lookup by
    looking at the unzipped archives and metadata file.

    TODO: implement zip dataset loading too
    """

    if mlcomp_root is None:
        try:
            mlcomp_root = os.environ['MLCOMP_DATASETS_HOME']
        except KeyError:
            raise ValueError("MLCOMP_DATASETS_HOME env variable is undefined")

    mlcomp_root = os.path.expanduser(mlcomp_root)
    mlcomp_root = os.path.abspath(mlcomp_root)
    mlcomp_root = os.path.normpath(mlcomp_root)

    if not os.path.exists(mlcomp_root):
        raise ValueError("Could not find folder: " + mlcomp_root)

    # dataset lookup
    if isinstance(name_or_id, numbers.Integral):
        # id lookup
        dataset_path = os.path.join(mlcomp_root, str(name_or_id))
    else:
        # assume name based lookup
        dataset_path = None
        expected_name_line = "name: " + name_or_id
        for dataset in os.listdir(mlcomp_root):
            metadata_file = os.path.join(mlcomp_root, dataset, 'metadata')
            if not os.path.exists(metadata_file):
                continue
            with open(metadata_file) as f:
                for line in f:
                    if line.strip() == expected_name_line:
                        dataset_path = os.path.join(mlcomp_root, dataset)
                        break
        if dataset_path is None:
            raise ValueError("Could not find dataset with metadata line: " +
                             expected_name_line)

    # loading the dataset metadata
    metadata = dict()
    metadata_file = os.path.join(dataset_path, 'metadata')
    if not os.path.exists(metadata_file):
        raise ValueError(dataset_path + ' is not a valid MLComp dataset')
    with open(metadata_file) as f:
        for line in f:
            if ":" in line:
                key, value = line.split(":", 1)
                metadata[key.strip()] = value.strip()

    format = metadata.get('format', 'unknow')
    loader = LOADERS.get(format)
    if loader is None:
        raise ValueError("No loader implemented for format: " + format)
    return loader(dataset_path, metadata, set_=set_, **kwargs)


import os
from astropy.io import ascii
from astropy.table import Table
from astropy.coordinates import Distance, Angle, SkyCoord
from astropy import units as u
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

import sys
# Change path so that we can import functions from the 'modules/' folder.
sys.path.insert(0, sys.path[0].replace('aux_', ''))
import readData
import MCs_data

def zDist(N):
    """
    This function generates a uniform spread of vertical distances, in the
    range (-z_dist, +z_dist).
    """

    # Define maximum vertical distance (in parsec)
    z_dist = 5000.
    # Generate N random z' vertical distances, in parsec.
    # To generate the *same* values each time the code is executed, fix the
    # random seed to any integer value.
    # np.random.seed(12345)
    z_prime = np.random.uniform(-z_dist, z_dist, N)

    return z_prime

def invertDist(incl, theta, ra_0, dec_0, D_0, ra, dec, z_prime):
    """
    Inverted distance in parsecs (D) from Eq (7) in
    van der Marel & Cioni (2001) using Eqs (1), (2), (3).
    """

    # Express everything in radians.
    incl, theta = np.deg2rad(incl), np.deg2rad(theta)
    ra_0, dec_0, ra, dec = ra_0.rad, dec_0.rad, np.deg2rad(ra), np.deg2rad(dec)

    # cos(rho)
    A = np.cos(dec) * np.cos(dec_0) * np.cos(ra - ra_0) +\
        np.sin(dec) * np.sin(dec_0)
    # sin(rho) * cos(phi)
    B = -np.cos(dec) * np.sin(ra - ra_0)
    # sin(rho) * sin(phi)
    C = np.sin(dec) * np.cos(dec_0) -\
        np.cos(dec) * np.sin(dec_0) * np.cos(ra - ra_0)

    # Eq (7)
    D = (z_prime - D_0.value * np.cos(incl)) /\
        (np.sin(incl) * (C * np.cos(theta) - B * np.sin(theta)) -
            A * np.cos(incl))

    return D

def rho_phi(ra, dec, glx_ctr):
    """
    Obtain the angular distance between (ra, dec) coordinates and the center
    of the galaxy (rho), and its position angle (phi).
    """

    # Store clusters' (ra, dec) coordinates in degrees.
    coords = SkyCoord(list(zip(*[ra, dec])), unit=(u.deg, u.deg))

    rho = coords.separation(glx_ctr)
    # Position angle between center and coordinates. This is the angle between
    # the positive y axis (North) counter-clockwise towards the negative x
    # axis (East).
    Phi = glx_ctr.position_angle(coords)
    # This is the angle measured counter-clockwise from the x positive axis
    # (West).
    phi = Phi + Angle('90d')

    return rho, phi

def xyz_coords(rho, phi, D_0, r_dist):
    '''
    Obtain coordinates in the (x,y,z) system of van der Marel & Cioni (2001),
    Eq (5).

    Values (x, y,z) returned in Kpc.
    '''
    d_kpc = Distance((10**(0.2 * (np.asarray(r_dist) + 5.))) / 1000.,
                     unit=u.kpc)

    x = d_kpc * np.sin(rho.radian) * np.cos(phi.radian)
    y = d_kpc * np.sin(rho.radian) * np.sin(phi.radian)
    z = D_0.kpc * u.kpc - d_kpc * np.cos(rho.radian)
    x, y, z = x.value, y.value, z.value

    return np.array([x, y, z])

def outData(gal, gal_data, dist_mod, e_dm):
    """
    Write data to output 'xxx_input_synth.dat' file ('xxx' stands for the
    processed galaxy.)
    """
    data = Table(
        [gal_data['Name'], gal_data['ra'], gal_data['dec'], dist_mod, e_dm,
         gal_data['log(age)']],
        names=['Name', 'ra', 'dec', 'dist_mod', 'e_dm', 'log(age)'])
    with open(gal.lower() + "_input_synth.dat", 'w') as f:
        ascii.write(data, f, format='fixed_width', delimiter=' ')

def inv_trans_eqs(x_p, y_p, z_p, theta, inc):
    """
    Inverse set of equations. Transform inclined plane system (x',y',z')
    into face on sky system (x,y,z).
    """
    x = x_p * np.cos(theta) - y_p * np.cos(inc) * np.sin(theta) -\
        z_p * np.sin(inc) * np.sin(theta)
    y = x_p * np.sin(theta) + y_p * np.cos(inc) * np.cos(theta) +\
        z_p * np.sin(inc) * np.cos(theta)
    z = -1. * y_p * np.sin(inc) + z_p * np.cos(inc)

    return x, y, z

def make_plot(gal_name, incl, theta, cl_xyz, dm):
    """
    Original link for plotting intersecting planes:
    http://stackoverflow.com/a/14825951/1391441
    """
    # Make plot.
    fig = plt.figure()
    ax = Axes3D(fig)

    # Placement 0, 0 is the bottom left, 1, 1 is the top right.
    ax.text2D(
        0.4, 0.95, r"${}:\;(\Theta, i) = ({}, {})$".format(
            gal_name, theta - 90., incl),
        transform=ax.transAxes, fontsize=15, color='red')

    # Express in radians for calculations.
    incl, theta = np.deg2rad(incl), np.deg2rad(theta)

    # Plot clusters.
    x_cl, y_cl, z_cl = cl_xyz
    SC = ax.scatter(x_cl, z_cl, y_cl, c=dm, s=50)

    min_X, max_X = min(x_cl) - 2., max(x_cl) + 2.
    min_Y, max_Y = min(y_cl) - 2., max(y_cl) + 2.
    min_Z, max_Z = min(z_cl) - 2., max(z_cl) + 2.

    # x,y plane.
    X, Y = np.meshgrid([min_X, max_X], [min_Y, max_Y])
    Z = np.zeros((2, 2))

    # Plot x,y plane.
    ax.plot_surface(X, Z, Y, color='gray', alpha=.1, linewidth=0, zorder=1)
    # Axis of x,y plane.
    # x axis.
    ax.plot([min_X, max_X], [0., 0.], [0., 0.], ls='--', c='k', zorder=4)
    # Arrow head pointing in the positive x direction.
    ax.quiver(max_X, 0., 0., max_X, 0., 0., arrow_length_ratio=.5,
              length=.1, color='k')
    ax.text(max_X, 0., -.5, 'x', 'x')
    # y axis.
    ax.plot([0., 0.], [0., 0.], [0., max_Y], ls='--', c='k')
    # Arrow head pointing in the positive y direction.
    ax.quiver(0., 0., max_Y, 0., 0., max_Y, arrow_length_ratio=.8,
              length=.1, color='k')
    ax.plot([0., 0.], [0., 0.], [min_Y, 0.], ls='--', c='k')
    ax.text(-.5, 0., max_Y, 'y', 'y')

    #
    # A plane is a*x+b*y+c*z+d=0, [a,b,c] is the normal.
    a, b, c, d = -1. * np.sin(theta) * np.sin(incl),\
        np.cos(theta) * np.sin(incl), np.cos(incl), 0.
    # print('a/c,b/c,1,d/c:', a / c, b / c, 1., d / c)
    # Rotated plane.
    X2_t, Y2_t = np.meshgrid([min_X, max_X], [0, max_Y])
    Z2_t = (-a * X2_t - b * Y2_t) / c
    X2_b, Y2_b = np.meshgrid([min_X, max_X], [min_Y, 0])
    Z2_b = (-a * X2_b - b * Y2_b) / c

    # Top half of first x',y' inclined plane.
    ax.plot_surface(X2_t, Z2_t, Y2_t, color='red', alpha=.1, lw=0, zorder=3)
    # Bottom half of inclined plane.
    ax.plot_surface(X2_t, Z2_b, Y2_b, color='red', alpha=.1, lw=0, zorder=-1)
    # Axis of x',y' plane.
    # x' axis.
    x_min, y_min, z_min = inv_trans_eqs(min_X, 0., 0., theta, incl)
    x_max, y_max, z_max = inv_trans_eqs(max_X, 0., 0., theta, incl)
    ax.plot([x_min, x_max], [z_min, z_max], [y_min, y_max], ls='--', c='b')
    # Arrow head pointing in the positive x' direction.
    ax.quiver(x_max, z_max, y_max, x_max, z_max, y_max, length=0.1,
              arrow_length_ratio=.7)
    ax.text(x_max, z_max, y_max - .5, "x'", 'x', color='b')
    # y' axis.
    x_min, y_min, z_min = inv_trans_eqs(0., min_Y, 0., theta, incl)
    x_max, y_max, z_max = inv_trans_eqs(0., max_Y, 0., theta, incl)
    ax.plot([x_min, x_max], [z_min, z_max], [y_min, y_max], ls='--', c='g')
    # Arrow head pointing in the positive y' direction.
    ax.quiver(x_max, z_max, y_max, x_max, z_max, y_max, length=0.1,
              arrow_length_ratio=.9, color='g')
    ax.text(x_max - .5, z_max, y_max, "y'", 'y', color='g')
    # # z' axis.
    # x_min, y_min, z_min = inv_trans_eqs(0., 0, min_Z, theta, incl)
    # x_max, y_max, z_max = inv_trans_eqs(0., 0, max_Z, theta, incl)
    # ax.plot([x_min, x_max], [z_min, z_max], [y_min, y_max], ls='--', c='y')
    # # Arrow head pointing in the positive z' direction.
    # ax.quiver(x_max, z_max, y_max, x_max, z_max, y_max, length=0.1,
    #           arrow_length_ratio=.9, color='y')
    # ax.text(x_max - .5, z_max, y_max, "z'", 'z', color='y')

    ax.set_xlabel('x (Kpc)')
    ax.set_ylabel('z (Kpc)')
    ax.set_ylim(max_Y, min_Y)
    ax.set_zlabel('y (Kpc)')
    plt.colorbar(SC, shrink=0.9, aspect=25)
    ax.axis('equal')
    ax.axis('tight')
    # This controls the initial orientation of the displayed 3D plot.
    # ‘elev’ stores the elevation angle in the z plane. ‘azim’ stores the
    # azimuth angle in the x,y plane.
    ax.view_init(elev=0., azim=-90.)
    plt.show()
    # plt.savefig()

def main():
    """
    """
    # Define inclination angles (i, Theta) (SMC first, LMC second).
    # 'Theta' is the PA (position angle) measured from the North (positive
    # y axis in van der Marel et al. 2002, Fig 3)
    rot_angles = ((60, 150.), (30, 140.))

    # Root path.
    r_path = os.path.realpath(__file__)[:-30]
    # Read input data for both galaxies from file (smc_data, lmc_data)
    gal_data = readData.main(r_path)

    for gal, gal_name in enumerate(['SMC', 'LMC']):
        print("Generating data for {}".format(gal_name))

        incl, Theta = rot_angles[gal]
        # 'theta' is the position angle measured from the West (positive
        # x axis), used by Eq (7) in van der Marel & Cioni (2001).
        theta = Theta + 90.

        # Center coordinates and distance for this galaxy.
        gal_center, D_0, e_gal_dist = MCs_data.MCs_data(gal)
        ra_0, dec_0 = gal_center.ra, gal_center.dec

        # Center coordinates for observed clusters in this galaxy.
        ra, dec = gal_data[gal]['ra'], gal_data[gal]['dec']

        # Generate N random vertical distances (z'), in parsec.
        z_prime = zDist(len(ra))

        # Distance to clusters in parsecs.
        D = invertDist(incl, theta, ra_0, dec_0, D_0, ra, dec, z_prime)

        # Convert to distance moduli.
        dist_mod = np.round(-5. + 5. * np.log10(D), 2)
        # This line below uses the actual distance moduli found by ASteCA.
        # dist_mod = gal_data[gal]['dist_mod']

        # Random errors for distance moduli.
        e_dm = np.round(np.random.uniform(.03, .09, len(ra)), 2)

        # Store data in output file.
        outData(gal_name, gal_data[gal], dist_mod, e_dm)
        print("Output data stored")

        # Obtain angular projected distance and position angle for the
        # clusters in the galaxy.
        rho, phi = rho_phi(ra, dec, gal_center)
        cl_xyz = xyz_coords(rho, phi, D_0, dist_mod)
        make_plot(gal_name, incl, theta, cl_xyz, dist_mod)
        print("Plot saved.")

if __name__ == '__main__':
    main()


import sys
import os
import datetime
import pickle
import numpy as np
import matplotlib.pyplot as plt
from pprint import pprint
import sqlite3
import calendar
from datetime import datetime

#id INTEGER PRIMARY KEY AUTOINCREMENT,
#sp2b_file TEXT,
#file_index INT,
#instr TEXT,
#instr_locn TEXT,
#particle_type TEXT,
#particle_dia FLOAT,
#unix_ts_utc FLOAT,
#actual_scat_amp FLOAT,
#actual_peak_pos INT,
#FF_scat_amp FLOAT,
#FF_peak_pos INT,
#FF_gauss_width FLOAT,
#zeroX_to_peak FLOAT,
#LF_scat_amp FLOAT,
#incand_amp FLOAT,
#lag_time_fit_to_incand FLOAT,
#LF_baseline_pct_diff FLOAT,
#rBC_mass_fg FLOAT,
#coat_thickness_nm FLOAT,
#coat_thickness_from_actual_scat_amp FLOAT
#UNIQUE (sp2b_file, file_index, instr)

#connect to database
conn = sqlite3.connect('C:/projects/dbs/SP2_data.db')
c = conn.cursor()
c2 = conn.cursor()

instrument = 'UBCSP2'
instrument_locn = 'WHI'
type_particle = 'incand'
start_date = '20110105'
end_date = '20120601'
lookup_file = 'C:/Users/Sarah Hanna/Documents/Data/WHI long term record/coatings/lookup_tables/coating_lookup_table_WHI_2012_UBCSP2-nc(2p26,1p26).lupckl'
rBC_density = 1.8
incand_sat = 3750

lookup = open(lookup_file, 'r')
lookup_table = pickle.load(lookup)
lookup.close()

c.execute('''SELECT * FROM SP2_coating_analysis''')
names = [description[0] for description in c.description]
pprint(names)

begin_data = calendar.timegm(datetime.strptime(start_date,'%Y%m%d').timetuple())
end_data = calendar.timegm(datetime.strptime(end_date,'%Y%m%d').timetuple())

def get_rBC_mass(incand_pk_ht, year):
	if year == 2012:
		rBC_mass = 0.003043*incand_pk_ht + 0.24826 #AD corrected linear calibration for UBCSP2 at WHI 2012
	if year == 2010:
		rBC_mass = 0.01081*incand_pk_ht - 0.32619  #AD corrected linear calibration for ECSP2 at WHI 2010
	return rBC_mass

def get_coating_thickness(BC_VED,scat_amp,coating_lookup_table):
	#get the coating thicknesses from the lookup table which is a dictionary of dictionaries, the 1st keyed with BC core size and the second being coating thicknesses keyed with calc scat amps
	core_diameters = sorted(coating_lookup_table.keys())
	prev_diameter = core_diameters[0]

	for core_diameter in core_diameters:
		if core_diameter > BC_VED:
			core_dia_to_use = prev_diameter
			break
		prev_diameter = core_diameter

	#now get the coating thickness for the scat_amp this is the coating thickness based on the raw scattering max
	scattering_amps = sorted(coating_lookup_table[core_dia_to_use].keys())
	prev_amp = scattering_amps[0]
	for scattering_amp in scattering_amps:
		if scat_amp < scattering_amp:
			scat_amp_to_use = prev_amp
			break
		prev_amp = scattering_amp

	scat_coating_thickness = coating_lookup_table[core_dia_to_use].get(scat_amp_to_use, np.nan) # returns value for the key, or none
	return scat_coating_thickness

LOG_EVERY_N = 10000

i = 0
for row in c.execute('''SELECT incand_amp, LF_scat_amp, unix_ts_utc, sp2b_file, file_index, instr FROM SP2_coating_analysis
WHERE instr=? and instr_locn=? and particle_type=? and incand_amp<? and unix_ts_utc>=? and unix_ts_utc<?''',
(instrument,instrument_locn,type_particle,incand_sat,begin_data,end_data)):
	incand_amp = row[0]
	LF_amp = row[1]
	event_time = datetime.utcfromtimestamp(row[2])
	file = row[3]
	index = row[4]
	instrt = row[5]

	rBC_mass = get_rBC_mass(incand_amp, event_time.year)
	if rBC_mass >= 0.25:
		rBC_VED = (((rBC_mass/(10**15*rBC_density))*6/3.14159)**(1/3.0))*10**7 #VED in nm with 10^15fg/g and 10^7nm/cm
		coat_th = get_coating_thickness(rBC_VED,LF_amp,lookup_table)
	else:
		rBC_VED = None
		coat_th = None

	c2.execute('''UPDATE SP2_coating_analysis SET coat_thickness_from_actual_scat_amp=? WHERE sp2b_file=? and file_index=? and instr=?''', (coat_th, file,index,instrt))

	i+=1
	if (i % LOG_EVERY_N) == 0:
		print 'record: ', i

conn.commit()
conn.close()


import logging
from typing import Dict, Optional

import numpy as np
import pandas as pd

from great_expectations.core import ExpectationConfiguration
from great_expectations.exceptions import InvalidExpectationConfigurationError
from great_expectations.execution_engine import (
    ExecutionEngine,
    PandasExecutionEngine,
    SparkDFExecutionEngine,
    SqlAlchemyExecutionEngine,
)
from great_expectations.expectations.core.expect_column_values_to_be_of_type import (
    _get_dialect_type_module,
    _native_type_type_map,
)
from great_expectations.expectations.expectation import ColumnMapExpectation
from great_expectations.expectations.registry import get_metric_kwargs
from great_expectations.expectations.util import render_evaluation_parameter_string
from great_expectations.render.renderer.renderer import renderer
from great_expectations.render.types import RenderedStringTemplateContent
from great_expectations.render.util import (
    num_to_str,
    parse_row_condition_string_pandas_engine,
    substitute_none_for_missing,
)
from great_expectations.validator.validation_graph import MetricConfiguration

logger = logging.getLogger(__name__)

try:
    import pyspark.sql.types as sparktypes
except ImportError as e:
    logger.debug(str(e))
    logger.debug(
        "Unable to load spark context; install optional spark dependency for support."
    )

class ExpectColumnValuesToBeInTypeList(ColumnMapExpectation):
    """
    Expect a column to contain values from a specified type list.

    expect_column_values_to_be_in_type_list is a \
    :func:`column_map_expectation <great_expectations.execution_engine.execution_engine.MetaExecutionEngine
    .column_map_expectation>` for typed-column backends,
    and also for PandasDataset where the column dtype provides an unambiguous constraints (any dtype except
    'object'). For PandasDataset columns with dtype of 'object' expect_column_values_to_be_of_type is a
    :func:`column_map_expectation <great_expectations.dataset.dataset.MetaDataset.column_map_expectation>` and will
    independently check each row's type.

    Args:
        column (str): \
            The column name.
        type_list (str): \
            A list of strings representing the data type that each column should have as entries. Valid types are
            defined by the current backend implementation and are dynamically loaded. For example, valid types for
            PandasDataset include any numpy dtype values (such as 'int64') or native python types (such as 'int'),
            whereas valid types for a SqlAlchemyDataset include types named by the current driver such as 'INTEGER'
            in most SQL dialects and 'TEXT' in dialects such as postgresql. Valid types for SparkDFDataset include
            'StringType', 'BooleanType' and other pyspark-defined type names.

    Keyword Args:
        mostly (None or a float between 0 and 1): \
            Return `"success": True` if at least mostly fraction of values match the expectation. \
            For more detail, see :ref:`mostly`.

    Other Parameters:
        result_format (str or None): \
            Which output mode to use: `BOOLEAN_ONLY`, `BASIC`, `COMPLETE`, or `SUMMARY`.
            For more detail, see :ref:`result_format <result_format>`.
        include_config (boolean): \
            If True, then include the expectation config as part of the result object. \
            For more detail, see :ref:`include_config`.
        catch_exceptions (boolean or None): \
            If True, then catch exceptions and include them as part of the result object. \
            For more detail, see :ref:`catch_exceptions`.
        meta (dict or None): \
            A JSON-serializable dictionary (nesting allowed) that will be included in the output without \
            modification. For more detail, see :ref:`meta`.

    Returns:
        An ExpectationSuiteValidationResult

        Exact fields vary depending on the values passed to :ref:`result_format <result_format>` and
        :ref:`include_config`, :ref:`catch_exceptions`, and :ref:`meta`.

    See also:
        :func:`expect_column_values_to_be_of_type \
        <great_expectations.dataset.dataset.Dataset.expect_column_values_to_be_of_type>`
    """

    # This dictionary contains metadata for display in the public gallery
    library_metadata = {
        "maturity": "production",
        "package": "great_expectations",
        "tags": ["core expectation", "column map expectation"],
        "contributors": ["@great_expectations"],
        "requirements": [],
    }

    map_metric = "column_values.in_type_list"

    success_keys = (
        "type_list",
        "mostly",
    )
    default_kwarg_values = {
        "type_list": None,
        "mostly": 1,
        "result_format": "BASIC",
        "include_config": True,
        "catch_exceptions": False,
    }

    def validate_configuration(self, configuration: Optional[ExpectationConfiguration]):
        super().validate_configuration(configuration)
        try:
            assert "type_list" in configuration.kwargs, "type_list is required"
            assert (
                isinstance(configuration.kwargs["type_list"], (list, dict))
                or configuration.kwargs["type_list"] is None
            ), "type_list must be a list or None"
            if isinstance(configuration.kwargs["type_list"], dict):
                assert (
                    "$PARAMETER" in configuration.kwargs["type_list"]
                ), 'Evaluation Parameter dict for type_list kwarg must have "$PARAMETER" key.'
        except AssertionError as e:
            raise InvalidExpectationConfigurationError(str(e))
        return True

    @classmethod
    @renderer(renderer_type="renderer.prescriptive")
    @render_evaluation_parameter_string
    def _prescriptive_renderer(
        cls,
        configuration=None,
        result=None,
        language=None,
        runtime_configuration=None,
        **kwargs
    ):
        runtime_configuration = runtime_configuration or {}
        include_column_name = runtime_configuration.get("include_column_name", True)
        include_column_name = (
            include_column_name if include_column_name is not None else True
        )
        styling = runtime_configuration.get("styling")
        params = substitute_none_for_missing(
            configuration.kwargs,
            ["column", "type_list", "mostly", "row_condition", "condition_parser"],
        )

        if params["type_list"] is not None:
            for i, v in enumerate(params["type_list"]):
                params["v__" + str(i)] = v
            values_string = " ".join(
                ["$v__" + str(i) for i, v in enumerate(params["type_list"])]
            )

            if params["mostly"] is not None:
                params["mostly_pct"] = num_to_str(
                    params["mostly"] * 100, precision=15, no_scientific=True
                )
                # params["mostly_pct"] = "{:.14f}".format(params["mostly"]*100).rstrip("0").rstrip(".")
                if include_column_name:
                    template_str = (
                        "$column value types must belong to this set: "
                        + values_string
                        + ", at least $mostly_pct % of the time."
                    )
                else:
                    template_str = (
                        "value types must belong to this set: "
                        + values_string
                        + ", at least $mostly_pct % of the time."
                    )
            else:
                if include_column_name:
                    template_str = (
                        "$column value types must belong to this set: "
                        + values_string
                        + "."
                    )
                else:
                    template_str = (
                        "value types must belong to this set: " + values_string + "."
                    )
        else:
            if include_column_name:
                template_str = "$column value types may be any value, but observed value will be reported"
            else:
                template_str = (
                    "value types may be any value, but observed value will be reported"
                )

        if params["row_condition"] is not None:
            (
                conditional_template_str,
                conditional_params,
            ) = parse_row_condition_string_pandas_engine(params["row_condition"])
            template_str = conditional_template_str + ", then " + template_str
            params.update(conditional_params)

        return [
            RenderedStringTemplateContent(
                **{
                    "content_block_type": "string_template",
                    "string_template": {
                        "template": template_str,
                        "params": params,
                        "styling": styling,
                    },
                }
            )
        ]

    def _validate_pandas(
        self,
        actual_column_type,
        expected_types_list,
    ):
        if expected_types_list is None:
            success = True
        else:
            comp_types = []
            for type_ in expected_types_list:
                try:
                    comp_types.append(np.dtype(type_).type)
                    comp_types.append(np.dtype(type_))
                except TypeError:
                    try:
                        pd_type = getattr(pd, type_)
                        if isinstance(pd_type, type):
                            comp_types.append(pd_type)
                    except AttributeError:
                        pass

                    try:
                        pd_type = getattr(pd.core.dtypes.dtypes, type_)
                        if isinstance(pd_type, type):
                            comp_types.append(pd_type)
                    except AttributeError:
                        pass

                native_type = _native_type_type_map(type_)
                if native_type is not None:
                    comp_types.extend(native_type)

            success = actual_column_type in comp_types

        return {
            "success": success,
            "result": {"observed_value": actual_column_type.type.__name__},
        }

    def _validate_sqlalchemy(
        self, actual_column_type, expected_types_list, execution_engine
    ):
        # Our goal is to be as explicit as possible. We will match the dialect
        # if that is possible. If there is no dialect available, we *will*
        # match against a top-level SqlAlchemy type.
        #
        # This is intended to be a conservative approach.
        #
        # In particular, we *exclude* types that would be valid under an ORM
        # such as "float" for postgresql with this approach

        if expected_types_list is None:
            success = True
        else:
            types = []
            type_module = _get_dialect_type_module(execution_engine=execution_engine)
            for type_ in expected_types_list:
                try:
                    type_class = getattr(type_module, type_)
                    types.append(type_class)
                except AttributeError:
                    logger.debug("Unrecognized type: %s" % type_)
            if len(types) == 0:
                logger.warning(
                    "No recognized sqlalchemy types in type_list for current dialect."
                )
            types = tuple(types)
            success = isinstance(actual_column_type, types)

        return {
            "success": success,
            "result": {"observed_value": type(actual_column_type).__name__},
        }

    def _validate_spark(
        self,
        actual_column_type,
        expected_types_list,
    ):
        if expected_types_list is None:
            success = True
        else:
            types = []
            for type_ in expected_types_list:
                try:
                    type_class = getattr(sparktypes, type_)
                    types.append(type_class)
                except AttributeError:
                    logger.debug("Unrecognized type: %s" % type_)
            if len(types) == 0:
                raise ValueError("No recognized spark types in expected_types_list")
            types = tuple(types)
            success = isinstance(actual_column_type, types)
        return {
            "success": success,
            "result": {"observed_value": type(actual_column_type).__name__},
        }

    def get_validation_dependencies(
        self,
        configuration: Optional[ExpectationConfiguration] = None,
        execution_engine: Optional[ExecutionEngine] = None,
        runtime_configuration: Optional[dict] = None,
    ):
        # This calls TableExpectation.get_validation_dependencies to set baseline dependencies for the aggregate version
        # of the expectation.
        # We need to keep this as super(ColumnMapExpectation, self), which calls
        # TableExpectation.get_validation_dependencies instead of ColumnMapExpectation.get_validation_dependencies.
        # This is because the map version of this expectation is only supported for Pandas, so we want the aggregate
        # version for the other backends.
        dependencies = super(ColumnMapExpectation, self).get_validation_dependencies(
            configuration, execution_engine, runtime_configuration
        )

        # Only PandasExecutionEngine supports the column map version of the expectation.
        if isinstance(execution_engine, PandasExecutionEngine):
            column_name = configuration.kwargs.get("column")
            expected_types_list = configuration.kwargs.get("type_list")
            metric_kwargs = get_metric_kwargs(
                configuration=configuration,
                metric_name="table.column_types",
                runtime_configuration=runtime_configuration,
            )
            metric_domain_kwargs = metric_kwargs.get("metric_domain_kwargs")
            metric_value_kwargs = metric_kwargs.get("metric_value_kwargs")
            table_column_types_configuration = MetricConfiguration(
                "table.column_types",
                metric_domain_kwargs=metric_domain_kwargs,
                metric_value_kwargs=metric_value_kwargs,
            )
            actual_column_types_list = execution_engine.resolve_metrics(
                [table_column_types_configuration]
            )[table_column_types_configuration.id]
            actual_column_type = [
                type_dict["type"]
                for type_dict in actual_column_types_list
                if type_dict["name"] == column_name
            ][0]

            # only use column map version if column dtype is object
            if (
                actual_column_type.type.__name__ == "object_"
                and expected_types_list is not None
            ):
                # this resets dependencies using  ColumnMapExpectation.get_validation_dependencies
                dependencies = super().get_validation_dependencies(
                    configuration, execution_engine, runtime_configuration
                )

        # this adds table.column_types dependency for both aggregate and map versions of expectation
        column_types_metric_kwargs = get_metric_kwargs(
            metric_name="table.column_types",
            configuration=configuration,
            runtime_configuration=runtime_configuration,
        )
        dependencies["metrics"]["table.column_types"] = MetricConfiguration(
            metric_name="table.column_types",
            metric_domain_kwargs=column_types_metric_kwargs["metric_domain_kwargs"],
            metric_value_kwargs=column_types_metric_kwargs["metric_value_kwargs"],
        )

        return dependencies

    def _validate(
        self,
        configuration: ExpectationConfiguration,
        metrics: Dict,
        runtime_configuration: dict = None,
        execution_engine: ExecutionEngine = None,
    ):
        column_name = configuration.kwargs.get("column")
        expected_types_list = configuration.kwargs.get("type_list")
        actual_column_types_list = metrics.get("table.column_types")
        actual_column_type = [
            type_dict["type"]
            for type_dict in actual_column_types_list
            if type_dict["name"] == column_name
        ][0]

        if isinstance(execution_engine, PandasExecutionEngine):
            # only PandasExecutionEngine supports map version of expectation and
            # only when column type is object
            if (
                actual_column_type.type.__name__ == "object_"
                and expected_types_list is not None
            ):
                # this calls ColumnMapMetric._validate
                return super()._validate(
                    configuration, metrics, runtime_configuration, execution_engine
                )
            return self._validate_pandas(
                actual_column_type=actual_column_type,
                expected_types_list=expected_types_list,
            )
        elif isinstance(execution_engine, SqlAlchemyExecutionEngine):
            return self._validate_sqlalchemy(
                actual_column_type=actual_column_type,
                expected_types_list=expected_types_list,
                execution_engine=execution_engine,
            )
        elif isinstance(execution_engine, SparkDFExecutionEngine):
            return self._validate_spark(
                actual_column_type=actual_column_type,
                expected_types_list=expected_types_list,
            )


################################################################################
# Peach - Computational Intelligence for Python
# Jose Alexandre Nalon
#
# This file: tutorial/linear-prediction.py
# Using neural networks to predict number sequences
################################################################################

# A neural network can be used to predict future values of a sequence of
# numbers. Wold's Decomposition Theorem stablishes that any sequence can be
# split in a regular and predictable part and an innovation process (which is
# discrete white noise, and thus impredictable). The goal of this tutorial is
# to show how to use the neural network implementation of Peach to do this.

# We import numpy for arrays and peach for the library. Actually, peach also
# imports the numpy module, but we want numpy in a separate namespace:
from numpy import *
import random
import peach as p

# First, we create the network, with only one layer with only one neuron in it.
# The neuron has many inputs and only one output. The activation function is the
# identity. This kind of neuron is usually known as ADALINE (Adaptive Linear
# Neuron, later Adaptive Linear Element). We use as learning algorithm the LMS
# algorithm.
N = 32
nn = p.FeedForward((N, 1), phi=p.Identity, lrule=p.LMS(0.05))

# The lists below will track the values of the sequence being predicted and of
# the error for plotting.
xlog = [ ]
ylog = [ ]
elog = [ ]

error = 1.
i = 0
x = zeros((N, 1), dtype=float)       # Input is a column-vector.
while i < 2000 and error > 1.e-10:

    # The sequence we will predict is the one generated by a cossinus. The next
    # value of the function is the desired output of the neuron. The neuron will
    # use past values to predict the unknown value. To spice things, we add some
    # gaussian noise (actually, it might help the convergence).
    d = cos(2.*pi/128. * i) + random.gauss(0., 0.01)

    # Here, we activate the network to calculate the prediction.
    y = nn(x)[0, 0]                 # Notice that we need to access the output
    error = abs(d - y)              # as a vector, since that's how the NN work.
    nn.learn(x, d)

    # We store the results to plot later.
    xlog.append(d)
    ylog.append(y)
    elog.append(error)

    # Here, we apply a delay in the sequence by shifting every value one
    # position back. We are using N (=32) samples to make the prediction, but
    # the code here makes no distinction and could be used with any number of
    # coefficients in the prediction. The last value of the sequence is put in
    # the [0] position of the vector.
    x[1:] = x[:-1]
    x[0] = d
    i = i + 1

# If the system has the plot package matplotlib, this tutorial tries to plot
# and save the convergence of synaptic weights and error. The plot is saved in
# the file ``linear-prediction.png``.
try:
    import pylab

    pylab.subplot(211)
    pylab.hold(True)
    pylab.grid(True)
    pylab.plot(array(xlog), 'b--')
    pylab.plot(array(ylog), 'g')
    pylab.plot(array(elog), 'r:')
    pylab.legend([ "$x$", "$y$", "$error$" ])

    pylab.subplot(212)
    pylab.grid(True)
    pylab.stem(arange(0, N), reshape(nn[0].weights, (N,)), "k-", "ko", "k-")
    pylab.xlim([0, N-1])
    pylab.savefig("linear-prediction.png")

except ImportError:
    print "After %d iterations:" % (len(elog),)
    print nn[0].weights


import numpy.fft as nfft
import numpy as np
import timeseries as ts
from scipy.stats import norm
# import pyfftw
import sys
#sys.path.append("/Users/yuhantang/CS207/TimeSeries/procs")
from .interface import *

def createfromlist(l):
    d = new_darray(len(l))
    for i in range(0,len(l)):
        darray_set(d,i,l[i])
    return d

def tsmaker(m, s, j):
    meta={}
    meta['order'] = int(np.random.choice([-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]))
    meta['blarg'] = int(np.random.choice([1, 2]))
    t = np.arange(0.0, 1.0, 0.01)
    v = norm.pdf(t, m, s) + j*np.random.randn(100)
    return meta, ts.TimeSeries(t, v)

def random_ts(a):
    t = np.arange(0.0, 1.0, 0.01)
    v = a*np.random.random(100)
    return ts.TimeSeries(t, v)

def stand(x, m, s):

    return (x-m)/s

def ccor(ts1, ts2):
    "given two standardized time series, compute their cross-correlation using FFT"

    # Get the next 2 th power 110 -> 128
    next_2 = int(2**np.ceil(np.log(len(ts1.values()))))

    #
    ts1_value = ts1.values()
    ts2_value = ts2.values()
    ts1_container,ts2_container = [],[]
    ts1_zero_container = [0]*len(ts1.values())
    ts2_zero_container = [0]*len(ts2.values())

    ts1_c_array,ts2_c_array = [None]*(len(ts1.values())*2),[None]*(len(ts2.values())*2)

    ts1_c_array[::2] = ts1_value
    ts1_c_array[1::2] = ts1_zero_container
    ts2_c_array[::2] = ts2_value
    ts2_c_array[1::2] = ts2_zero_container

    for i in range(len(ts1_c_array)+1,next_2*2):
        ts1_c_array.append(np.double(0))

    for i in range(len(ts2_c_array)+1,next_2*2):
        ts2_c_array.append(np.double(0))
    ts1_c_array.insert(0,0)
    ts2_c_array.insert(0,0)

    ts1_c_array = createfromlist(np.double(ts1_c_array))
    ts2_c_array = createfromlist(np.double(ts2_c_array))

    four1(ts1_c_array,next_2,1)
    four1(ts2_c_array,next_2,1)

    for i in range(len(ts2.values())*2+1):
        ts1_container.append(darray_get(ts1_c_array,i))
    for j in range(len(ts1.values())*2+1):
        ts2_container.append(darray_get(ts2_c_array,j))

    ts1_fft = np.asarray(ts1_container[1::2]) + 1j * np.asarray(ts1_container[2::2])
    ts2_fft = np.asarray(ts2_container[1::2]) + 1j * np.asarray(ts2_container[2::2])
    ts1_fft = ts1_fft[:len(ts1)+1]
    ts2_fft = ts2_fft[:len(ts2)+1]

    # ifft part

    ts1_ts2_conj = ts1_fft * np.conj(ts2_fft)
    ts1_ts2_ifft_container = [0]*len(ts1_ts2_conj)*2
    ts1_ts2_ifft_container[::2] = ts1_ts2_conj.real
    ts1_ts2_ifft_container[1::2] = ts1_ts2_conj.imag

    for i in range(len(ts1_ts2_conj)+1, next_2 *2):
        ts1_ts2_ifft_container.append(0)

    ts1_ts2_ifft_container.insert(0,0)

    ts1_ts2_ifft_container = createfromlist(ts1_ts2_ifft_container)

    four1(ts1_ts2_ifft_container, next_2, -1)

    ts1_ts2_ifft_container_python = []

    for i in range(len(ts1_ts2_conj)*2+1):
        ts1_ts2_ifft_container_python.append(darray_get(ts1_ts2_ifft_container,i))

    ccor_value = np.asarray(ts1_ts2_ifft_container_python[1::2])

    return 1/len(ts1) * ccor_value

def max_corr_at_phase(ts1, ts2):
    ccorts = ccor(ts1, ts2)
    idx = np.argmax(ccorts)
    maxcorr = ccorts[idx]
    return idx, maxcorr

#The equation for the kernelized cross correlation is given at
#http://www.cs.tufts.edu/~roni/PUB/ecml09-tskernels.pdf
#normalize the kernel there by np.sqrt(K(x,x)K(y,y)) so that the correlation
#of a time series with itself is 1.
def kernel_corr(ts1, ts2, mult=1):
    "compute a kernelized correlation so that we can get a real distance"
    #your code here.
    cross_correlation = ccor(ts1, ts2) * mult
    corr_ts1, corr_ts2 = ccor(ts1, ts1) * mult, ccor(ts2, ts2) * mult
    return np.sum(np.exp(cross_correlation))/np.sqrt(np.sum(np.exp(corr_ts1))*np.sum(np.exp(corr_ts2)))

#this is for a quick and dirty test of these functions
#you might need to add procs to pythonpath for this to work
if __name__ == "__main__":
    print("HI")
    _, t1 = tsmaker(0.5, 0.1, 0.01)
    _, t2 = tsmaker(0.5, 0.1, 0.01)
    print(t1.mean(), t1.std(), t2.mean(), t2.std())
    import matplotlib.pyplot as plt
    plt.plot(t1)
    plt.plot(t2)
    plt.show()
    standts1 = stand(t1, t1.mean(), t1.std())
    standts2 = stand(t2, t2.mean(), t2.std())
    #print(type(standts1),'this is the type=================*********')
    #assert 1 == 2
    idx, mcorr = max_corr_at_phase(standts1, standts2)
    print(idx, mcorr)
    sumcorr = kernel_corr(standts1, standts2, mult=10)
    print(sumcorr)
    t3 = random_ts(2)
    t4 = random_ts(3)
    plt.plot(t3)
    plt.plot(t4)
    plt.show()
    standts3 = stand(t3, t3.mean(), t3.std())
    standts4 = stand(t4, t4.mean(), t4.std())
    idx, mcorr = max_corr_at_phase(standts3, standts4)
    print(idx, mcorr)
    sumcorr = kernel_corr(standts3, standts4, mult=10)
    print(sumcorr)


# Working with Bag of Words
#---------------------------------------
#
# In this example, we will download and preprocess the ham/spam
#  text data.  We will then use a one-hot-encoding to make a
#  bag of words set of features to use in logistic regression.
#
# We will use these one-hot-vectors for logistic regression to
#  predict if a text is spam or ham.

import tensorflow as tf
import matplotlib.pyplot as plt
import os
import numpy as np
import csv
import string
import requests
import io
from zipfile import ZipFile
from tensorflow.contrib import learn
from tensorflow.python.framework import ops
ops.reset_default_graph()

# Start a graph session
sess = tf.Session()

# Check if data was downloaded, otherwise download it and save for future use
save_file_name = os.path.join('temp','temp_spam_data.csv')
if os.path.isfile(save_file_name):
    text_data = []
    with open(save_file_name, 'r') as temp_output_file:
        reader = csv.reader(temp_output_file)
        for row in reader:
            text_data.append(row)
else:
    zip_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip'
    r = requests.get(zip_url)
    z = ZipFile(io.BytesIO(r.content))
    file = z.read('SMSSpamCollection')
    # Format Data
    text_data = file.decode()
    text_data = text_data.encode('ascii',errors='ignore')
    text_data = text_data.decode().split('\n')
    text_data = [x.split('\t') for x in text_data if len(x)>=1]

    # And write to csv
    with open(save_file_name, 'w') as temp_output_file:
        writer = csv.writer(temp_output_file)
        writer.writerows(text_data)

texts = [x[1] for x in text_data]
target = [x[0] for x in text_data]

# Relabel 'spam' as 1, 'ham' as 0
target = [1 if x=='spam' else 0 for x in target]

# Normalize text
# Lower case
texts = [x.lower() for x in texts]

# Remove punctuation
texts = [''.join(c for c in x if c not in string.punctuation) for x in texts]

# Remove numbers
texts = [''.join(c for c in x if c not in '0123456789') for x in texts]

# Trim extra whitespace
texts = [' '.join(x.split()) for x in texts]

# Plot histogram of text lengths
text_lengths = [len(x.split()) for x in texts]
text_lengths = [x for x in text_lengths if x < 50]
plt.hist(text_lengths, bins=25)
plt.title('Histogram of # of Words in Texts')

# Choose max text word length at 25
sentence_size = 25
min_word_freq = 3

# Setup vocabulary processor
vocab_processor = learn.preprocessing.VocabularyProcessor(sentence_size, min_frequency=min_word_freq)

# Have to fit transform to get length of unique words.
vocab_processor.fit_transform(texts)
embedding_size = len(vocab_processor.vocabulary_)

# Split up data set into train/test
train_indices = np.random.choice(len(texts), round(len(texts)*0.8), replace=False)
test_indices = np.array(list(set(range(len(texts))) - set(train_indices)))
texts_train = [x for ix, x in enumerate(texts) if ix in train_indices]
texts_test = [x for ix, x in enumerate(texts) if ix in test_indices]
target_train = [x for ix, x in enumerate(target) if ix in train_indices]
target_test = [x for ix, x in enumerate(target) if ix in test_indices]

# Setup Index Matrix for one-hot-encoding
identity_mat = tf.diag(tf.ones(shape=[embedding_size]))

# Create variables for logistic regression
A = tf.Variable(tf.random_normal(shape=[embedding_size,1]))
b = tf.Variable(tf.random_normal(shape=[1,1]))

# Initialize placeholders
x_data = tf.placeholder(shape=[sentence_size], dtype=tf.int32)
y_target = tf.placeholder(shape=[1, 1], dtype=tf.float32)

# Text-Vocab Embedding
x_embed = tf.nn.embedding_lookup(identity_mat, x_data)
x_col_sums = tf.reduce_sum(x_embed, 0)

# Declare model operations
x_col_sums_2D = tf.expand_dims(x_col_sums, 0)
model_output = tf.add(tf.matmul(x_col_sums_2D, A), b)

# Declare loss function (Cross Entropy loss)
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(model_output, y_target))

# Prediction operation
prediction = tf.sigmoid(model_output)

# Declare optimizer
my_opt = tf.train.GradientDescentOptimizer(0.001)
train_step = my_opt.minimize(loss)

# Intitialize Variables
init = tf.initialize_all_variables()
sess.run(init)

# Start Logistic Regression
print('Starting Training Over {} Sentences.'.format(len(texts_train)))
loss_vec = []
train_acc_all = []
train_acc_avg = []
for ix, t in enumerate(vocab_processor.fit_transform(texts_train)):
    y_data = [[target_train[ix]]]

    sess.run(train_step, feed_dict={x_data: t, y_target: y_data})
    temp_loss = sess.run(loss, feed_dict={x_data: t, y_target: y_data})
    loss_vec.append(temp_loss)

    if (ix+1)%10==0:
        print('Training Observation #' + str(ix+1) + ': Loss = ' + str(temp_loss))

    # Keep trailing average of past 50 observations accuracy
    # Get prediction of single observation
    [[temp_pred]] = sess.run(prediction, feed_dict={x_data:t, y_target:y_data})
    # Get True/False if prediction is accurate
    train_acc_temp = target_train[ix]==np.round(temp_pred)
    train_acc_all.append(train_acc_temp)
    if len(train_acc_all) >= 50:
        train_acc_avg.append(np.mean(train_acc_all[-50:]))

# Get test set accuracy
print('Getting Test Set Accuracy For {} Sentences.'.format(len(texts_test)))
test_acc_all = []
for ix, t in enumerate(vocab_processor.fit_transform(texts_test)):
    y_data = [[target_test[ix]]]

    if (ix+1)%50==0:
        print('Test Observation #' + str(ix+1))

    # Keep trailing average of past 50 observations accuracy
    # Get prediction of single observation
    [[temp_pred]] = sess.run(prediction, feed_dict={x_data:t, y_target:y_data})
    # Get True/False if prediction is accurate
    test_acc_temp = target_test[ix]==np.round(temp_pred)
    test_acc_all.append(test_acc_temp)

print('\nOverall Test Accuracy: {}'.format(np.mean(test_acc_all)))

# Plot training accuracy over time
plt.plot(range(len(train_acc_avg)), train_acc_avg, 'k-', label='Train Accuracy')
plt.title('Avg Training Acc Over Past 50 Generations')
plt.xlabel('Generation')
plt.ylabel('Training Accuracy')
plt.show()


"""Base and mixin classes for instance reduction techniques"""
# Author: Dayvid Victor <dvro@cin.ufpe.br>
# License: BSD Style
import warnings
from abc import ABCMeta, abstractmethod

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.neighbors.classification import KNeighborsClassifier

from sklearn.utils import check_array
from sklearn.externals import six

class InstanceReductionWarning(UserWarning):
    pass

# Make sure that NeighborsWarning are displayed more than once
warnings.simplefilter("always", InstanceReductionWarning)

class InstanceReductionBase(six.with_metaclass(ABCMeta, BaseEstimator)):

    """Base class for instance reduction estimators."""

    @abstractmethod
    def __init__(self):
        pass

class InstanceReductionMixin(InstanceReductionBase, ClassifierMixin):

    """Mixin class for all instance reduction techniques"""

    def set_classifier(self):
        """Sets the classified to be used in the instance reduction process
            and classification.

        Parameters
        ----------
        classifier : classifier, following the KNeighborsClassifier style
            (default = KNN)

        y : array-like, shape = [n_samples]
            Labels for X.

        Returns
        -------
        P : array-like, shape = [indeterminated, n_features]
            Resulting training set.

        q : array-like, shape = [indertaminated]
            Labels for P
        """

        self.classifier = classifier

    def reduce_data(self, X, y):
        """Perform the instance reduction procedure on the given training data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training set.0

        y : array-like, shape = [n_samples]
            Labels for X.

        Returns
        -------
        X_ : array-like, shape = [indeterminated, n_features]
            Resulting training set.

        y_ : array-like, shape = [indertaminated]
            Labels for X_
        """
        pass

    def fit(self, X, y, reduce_data=True):
        """
        Fit the InstanceReduction model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.
            Note that centroid shrinking cannot be used with sparse matrices.
        y : array, shape = [n_samples]
            Target values (integers)
        reduce_data : bool, flag indicating if the reduction would be performed
        """
        self.X = X
        self.y = y

        if reduce_data:
            self.reduce_data(X, y)

        return self

    def predict(self, X, n_neighbors=1):
        """Perform classification on an array of test vectors X.

        The predicted class C for each sample in X is returned.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = [n_samples]

        Notes
        -----
        The default prediction is using KNeighborsClassifier, if the
        instance reducition algorithm is to be performed with another
        classifier, it should be explicited overwritten and explained
        in the documentation.
        """
        X = check_array(X)
        if not hasattr(self, "X_") or self.X_ is None:
            raise AttributeError("Model has not been trained yet.")

        if not hasattr(self, "y_") or self.y_ is None:
            raise AttributeError("Model has not been trained yet.")

        if self.classifier == None:
            self.classifier = KNeighborsClassifier(n_neighbors=n_neighbors)

        self.classifier.fit(self.X_, self.y_)
        return self.classifier.predict(X)

    def predict_proba(self, X):
        """Return probability estimates for the test data X.
        after a given prototype selection algorithm.

        Parameters
        ----------
        X : array, shape = (n_samples, n_features)
            A 2-D array representing the test points.

        Returns
        -------
        p : array of shape = [n_samples, n_classes], or a list of n_outputs
        of such arrays if n_outputs > 1.
        The class probabilities of the input samples. Classes are ordered
        by lexicographic order.
        """
        self.classifier.fit(self.X_, self.y_)
        return self.classifier.predict_proba(X)