import numpy as np
import matplotlib.pyplot as plt
import urllib2

#%matplotlib inline

sample_size = 1000
np.random.seed(1)
url = ('https://raw.githubusercontent.com/Upward-Spiral-Science'
       '/data/master/syn-density/output.csv')
data = urllib2.urlopen(url)
csv = np.genfromtxt(data, delimiter=",")[1:]

csv_rand = None
for i in range (1, sample_size):
	#Randomly sample from dataset
	a = np.random.permutation(np.arange(csv.shape[0]))[:100]
	csv_rand_sample = csv[a]

	# Normalize
	mean_unmask = np.mean(csv_rand_sample[:,3])
	std_unmask = np.std(csv_rand_sample[:,3])
	csv_rand_sample[:,3] = (csv_rand_sample[:,3]-mean_unmask)/std_unmask

	#Stack matrix
	if i == 1:
		csv_rand = csv_rand_sample
	else:
		csv_rand = np.dstack((csv_rand,csv_rand_sample))

#Average across random samples
csv_rand = np.mean(csv_rand,axis=2)

#Independence Assumption
covar = np.cov(csv_rand_sample)

plt.figure(figsize=(7,7))
plt.imshow(covar)
plt.title('Covariance of Synapse Density dataset')
plt.colorbar()
plt.show()

diag = covar.diagonal()*np.eye(covar.shape[0])
hollow = covar-diag
d_det = np.linalg.slogdet(diag)[1]
h_det = np.linalg.slogdet(hollow)[1]
print d_det
print h_det

plt.figure(figsize=(11,8))
plt.subplot(121)
plt.imshow(diag)
plt.clim([0, np.max(covar)])
plt.title('Determinant of on-diagonal: ' + str(d_det))
plt.subplot(122)
plt.imshow(hollow)
plt.clim([0, np.max(covar)])
plt.title('Determinant of off-diagonal: ' + str(h_det))
plt.show()

print "Ratio of on and off-diagonal determinants: " + str(d_det/h_det)


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
========================================================================
Gaussian Processes regression: goodness-of-fit on the 'diabetes' dataset
========================================================================

In this example, we fit a Gaussian Process model onto the diabetes
dataset.

We determine the correlation parameters with maximum likelihood
estimation (MLE). We use an anisotropic squared exponential
correlation model with a constant regression model. We also use a
nugget of 1e-2 to account for the (strong) noise in the targets.

We compute a cross-validation estimate of the coefficient of
determination (R2) without reperforming MLE, using the set of correlation
parameters found on the whole dataset.
"""
print(__doc__)

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
# Licence: BSD 3 clause

from sklearn import datasets
from sklearn.gaussian_process import GaussianProcess
from sklearn.cross_validation import cross_val_score, KFold

# Load the dataset from scikit's data sets
diabetes = datasets.load_diabetes()
X, y = diabetes.data, diabetes.target

# Instanciate a GP model
gp = GaussianProcess(regr='constant', corr='absolute_exponential',
                     theta0=[1e-4] * 10, thetaL=[1e-12] * 10,
                     thetaU=[1e-2] * 10, nugget=1e-2, optimizer='Welch')

# Fit the GP model to the data performing maximum likelihood estimation
gp.fit(X, y)

# Deactivate maximum likelihood estimation for the cross-validation loop
gp.theta0 = gp.theta_  # Given correlation parameter = MLE
gp.thetaL, gp.thetaU = None, None  # None bounds deactivate MLE

# Perform a cross-validation estimate of the coefficient of determination using
# the cross_validation module using all CPUs available on the machine
K = 20  # folds
R2 = cross_val_score(gp, X, y=y, cv=KFold(y.size, K), n_jobs=1).mean()
print("The %d-Folds estimate of the coefficient of determination is R2 = %s"
      % (K, R2))


# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests of the DataFrame class."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib.learn.python import learn
from tensorflow.contrib.learn.python.learn.tests.dataframe import mocks
from tensorflow.python.framework import dtypes
from tensorflow.python.platform import test

def setup_test_df():
  """Create a dataframe populated with some test columns."""
  df = learn.DataFrame()
  df["a"] = learn.TransformedSeries(
      [mocks.MockSeries("foobar", mocks.MockTensor("Tensor a", dtypes.int32))],
      mocks.MockTwoOutputTransform("iue", "eui", "snt"), "out1")
  df["b"] = learn.TransformedSeries(
      [mocks.MockSeries("foobar", mocks.MockTensor("Tensor b", dtypes.int32))],
      mocks.MockTwoOutputTransform("iue", "eui", "snt"), "out2")
  df["c"] = learn.TransformedSeries(
      [mocks.MockSeries("foobar", mocks.MockTensor("Tensor c", dtypes.int32))],
      mocks.MockTwoOutputTransform("iue", "eui", "snt"), "out1")
  return df

class DataFrameTest(test.TestCase):
  """Test of `DataFrame`."""

  def test_create(self):
    df = setup_test_df()
    self.assertEqual(df.columns(), frozenset(["a", "b", "c"]))

  def test_select_columns(self):
    df = setup_test_df()
    df2 = df.select_columns(["a", "c"])
    self.assertEqual(df2.columns(), frozenset(["a", "c"]))

  def test_exclude_columns(self):
    df = setup_test_df()
    df2 = df.exclude_columns(["a", "c"])
    self.assertEqual(df2.columns(), frozenset(["b"]))

  def test_get_item(self):
    df = setup_test_df()
    c1 = df["b"]
    self.assertEqual(
        mocks.MockTensor("Mock Tensor 2", dtypes.int32), c1.build())

  def test_del_item_column(self):
    df = setup_test_df()
    self.assertEqual(3, len(df))
    del df["b"]
    self.assertEqual(2, len(df))
    self.assertEqual(df.columns(), frozenset(["a", "c"]))

  def test_set_item_column(self):
    df = setup_test_df()
    self.assertEqual(3, len(df))
    col1 = mocks.MockSeries("QuackColumn",
                            mocks.MockTensor("Tensor ", dtypes.int32))
    df["quack"] = col1
    self.assertEqual(4, len(df))
    col2 = df["quack"]
    self.assertEqual(col1, col2)

  def test_set_item_column_multi(self):
    df = setup_test_df()
    self.assertEqual(3, len(df))
    col1 = mocks.MockSeries("QuackColumn", [])
    col2 = mocks.MockSeries("MooColumn", [])
    df["quack", "moo"] = [col1, col2]
    self.assertEqual(5, len(df))
    col3 = df["quack"]
    self.assertEqual(col1, col3)
    col4 = df["moo"]
    self.assertEqual(col2, col4)

  def test_set_item_pandas(self):
    # TODO(jamieas)
    pass

  def test_set_item_numpy(self):
    # TODO(jamieas)
    pass

  def test_build(self):
    df = setup_test_df()
    result = df.build()
    expected = {
        "a": mocks.MockTensor("Mock Tensor 1", dtypes.int32),
        "b": mocks.MockTensor("Mock Tensor 2", dtypes.int32),
        "c": mocks.MockTensor("Mock Tensor 1", dtypes.int32)
    }
    self.assertEqual(expected, result)

if __name__ == "__main__":
  test.main()


# utils
# Utility functions for handling data
#
# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
# Created:  Thu Feb 26 17:47:35 2015 -0500
#
# Copyright (C) 2015 District Data Labs
# For license information, see LICENSE.txt
#
# ID: utils.py [] benjamin@bengfort.com $

"""
Utility functions for handling data
"""

##########################################################################
## Imports
##########################################################################

import os
import csv
import time
import json
import numpy as np

from sklearn.datasets.base import Bunch

##########################################################################
## Module Constants
##########################################################################

SKL_DATA = "SCIKIT_LEARN_DATA"
BASE_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
DATA_DIR = os.path.join(BASE_DIR, "data")
CODE_DIR = os.path.join(BASE_DIR, "code")

##########################################################################
## Helper Functions
##########################################################################

def timeit(func):
    """
    Returns how long a function took to execute, along with the output
    """

    def wrapper(*args, **kwargs):
        start  = time.time()
        result = func(*args, **kwargs)
        return result, time.time() - start

    return timeit

##########################################################################
## Dataset Loading
##########################################################################

def get_data_home(data_home=None):
    """
    Returns the path of the data directory
    """
    if data_home is None:
        data_home = os.environ.get(SKL_DATA, DATA_DIR)

    data_home = os.path.expanduser(data_home)
    if not os.path.exists(data_home):
        os.makedirs(data_home)

    return data_home

def load_data(path, descr=None, target_index=-1):
    """
    Returns a scklearn dataset Bunch which includes several important
    attributes that are used in modeling:

        data: array of shape n_samples * n_features
        target: array of length n_samples
        feature_names: names of the features
        target_names: names of the targets
        filenames: names of the files that were loaded
        DESCR: contents of the readme

    This data therefore has the look and feel of the toy datasets.

    Pass in a path usually just the name of the location in the data dir.
    It will be joined with the result of `get_data_home`. The contents are:

        path
            - abalone.names     # The file to load into DESCR
            - meta.json     # A file containing metadata to load
            - dataset.txt   # The numpy loadtxt file
            - dataset.csv   # The pandas read_csv file

    You can specify another descr, another feature_names, and whether or
    not the dataset has a header row. You can also specify the index of the
    target, which by default is the last item in the row (-1)
    """

    root          = os.path.join(get_data_home(), path)
    filenames     = {
        'meta': os.path.join(root, 'meta.json'),
        'rdme': os.path.join(root, 'abalone.names'),
        'data': os.path.join(root, 'dataset.csv'),
    }

    target_names  = None
    feature_names = None
    DESCR         = None

    with open(filenames['meta'], 'r') as f:
        meta = json.load(f)
        target_names  = meta['target_names']
        feature_names = meta['feature_names']

    with open(filenames['rdme'], 'r') as f:
        DESCR = f.read()

    # skip header from csv, load data
    dataset = np.loadtxt(filenames['data'], delimiter=',', skiprows=1)
    data    = None
    target  = None

    # Target assumed to be either last or first row
    if target_index == -1:
        data   = dataset[:,0:-1]
        target = dataset[:,-1]
    elif target_index == 0:
        data   = dataset[:,1:]
        target = dataset[:,0]
    else:
        raise ValueError("Target index must be either -1 or 0")

    return Bunch(data=data,
                 target=target,
                 filenames=filenames,
                 target_names=target_names,
                 feature_names=feature_names,
                 DESCR=DESCR)

def load_abalone():
    return load_data('abalone')


import os
os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
import pandas as pd
import matplotlib
import numpy as np
import math
import matplotlib.pyplot as plt
from sklearn.preprocessing import Imputer
from sklearn.cross_validation import train_test_split
from sklearn import preprocessing
import plotly.tools as tls
import pandas as pd
from sqlalchemy import create_engine # database connection
import datetime as dt
import io
import logging
import plotly.plotly as py # interactive graphing
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
from plotly.graph_objs import Bar, Scatter, Marker, Layout
from heraspy.model import HeraModel
np.random.seed(1337)
import theano
import keras
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model,model_from_yaml
from keras.layers import Input, Dense, GRU, LSTM, TimeDistributed, Masking,merge
from model import *
import argparse
import sys

if __name__ == "__main__":

    t_start = dt.datetime.now()
    parser = argparse.ArgumentParser(prog='Weighted Model')
    parser.add_argument('-t','--table',required=True)
    args = parser.parse_args()

    ####################################DATA SOURCE################################
    table = vars(args)['table']
    # table = 'data_trim'
    # rsl_file = './data/gs_results_trim.csv'
    # rsl_file = './data/psql_data_trim.csv'

    # table = 'data_little_enc'
    # rsl_file = './data/gs_results_little.csv'

    # table = 'data_more'
    # rsl_file = './data/gs_results_more.csv'
    # table = 'auth'
    # rsl_file = './data/auth.csv'

    events_tbl = 'event'
    events_tbl = None
    rsl_file = './data/psql_{table}.csv'.format(table=table)
    ################################################################################

    print "Commencing..."
    data_dir = './data/'
    evt_name = 'Featurespace_events_output.csv'
    auth_name = 'Featurespace_auths_output.csv'
    db_name = 'c1_agg.db'

    address = "postgresql+pg8000://script@localhost:5432/ccfd"
    # disk_engine = create_engine('sqlite:///'+data_dir+db_name,convert_unicode=True)
    # disk_engine.raw_connection().connection.text_factory = str
    disk_engine = create_engine(address)

    #######################Settings#############################################
    samples_per_epoch = trans_num_table(table,disk_engine,mode='train',trans_mode='train')
    # epoch_limit = 10000
    # samples_per_epoch = epoch_limit
    # user_sample_size = 8000

    epoch_limit = samples_per_epoch
    user_sample_size = None

    nb_epoch = 300
    fraud_w_list = [1000.]

    ##########ENCODERS CONF
    tbl_src = 'auth'
    # tbl_src = table
    tbl_evnt = 'event'
    ##################################

    batch_size = 300
    batch_size_val = 1000
    print "SAMPLES per epoch:",samples_per_epoch
    print "User sample size:",user_sample_size
    print 'sequence length size',batch_size
    # samples_per_epoch = 1959
    # table = 'data_trim'
    # samples_per_epoch = 485

    lbl_pad_val = 2
    pad_val = 0

    # dropout_W_list = [0.3]
    dropout_W_list = [0.4,0.5,0.6,0.7]
    # dropout_W_list = [0.15,0.3,0.4,0.8]

    input_dim = 44
    hid_dims = [320]
    num_l = [7]
    lr_s = [2.5e-4]
    # lr_s = [1.25e-4,6e-5]
    # lr_s = [1e-2,1e-3,1e-4]
    # lr_s = [1e-1,1e-2,1e-3]
    num_opt = 1
    opts = lambda x,lr:[keras.optimizers.RMSprop(lr=lr, rho=0.9, epsilon=1e-08),
                    # keras.optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08),
                    # keras.optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004)
                    ][x]
    # add_info = str(int(seq_len_param))+'_class_w_'+str(fraud_w)

    print 'Populating encoders'

    path_encoders ='./data/encoders/{tbl_src}+{tbl_evnt}'.format(tbl_src=tbl_src,tbl_evnt=tbl_evnt)
    if os.path.exists(path_encoders):
        encoders = load_encoders(path_encoders)
    else:
        encoders = populate_encoders_scale(tbl_src,disk_engine,tbl_evnt)
        with open(path_encoders, 'wb') as output:
            pickle.dump(encoders, output, pickle.HIGHEST_PROTOCOL)
            print 'ENCODERS SAVED to {path}!'.format(path=path_encoders)

    # sys.exit()
    gru_dict = {}
    lstm_dict = {}
    for fraud_w in fraud_w_list:
        add_info = 'Mask=pad_class_w_'+str(fraud_w)+'ES-OFF'
        class_weight = {0 : 1.,
            1: fraud_w,
            2: 0.}
        for dropout_W in dropout_W_list:
            for hidden_dim in hid_dims:
            # gru
                for opt_id in range(num_opt):
                    for lr in lr_s:
                        optimizer = opts(opt_id,lr)
                        for num_layers in num_l:
                            for rnn in ['gru']:

                                short_title = 'bi_'+rnn.upper()+'_'+str(hidden_dim)+'_'+str(num_layers)+'_DO-'+str(dropout_W)+'_w'+str(class_weight[1])
                                title = 'Bidirectional_Class'+str(class_weight[1])+'_'+rnn.upper()+'_'+str(hidden_dim)+'_'+str(num_layers)+'_'+str(type(optimizer).__name__)+'_'+str(lr)+'_epochs_'+str(nb_epoch)+'_DO-'+str(dropout_W)
                                print title
                                input_layer = Input(shape=(int(seq_len_param), input_dim),name='main_input')
                                mask = Masking(mask_value=pad_val)(input_layer)
                                x = mask
                                for i in range(num_layers):
                                    if rnn == 'gru':
                                        prev_frw = GRU(hidden_dim,#input_length=50,
                                                            return_sequences=True,go_backwards=False,stateful=False,
                                                            unroll=False,consume_less='gpu',
                                                            init='glorot_uniform', inner_init='orthogonal', activation='tanh',
                                                    inner_activation='hard_sigmoid', W_regularizer=None, U_regularizer=None,
                                                    b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                        prev_bck = GRU(hidden_dim,#input_length=50,
                                                            return_sequences=True,go_backwards=True,stateful=False,
                                                            unroll=False,consume_less='gpu',
                                                            init='glorot_uniform', inner_init='orthogonal', activation='tanh',
                                                    inner_activation='hard_sigmoid', W_regularizer=None, U_regularizer=None,
                                                    b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                    else:
                                        prev_frw = LSTM(hidden_dim, return_sequences=True,go_backwards=False,stateful=False,
                                            init='glorot_uniform', inner_init='orthogonal',
                                            forget_bias_init='one', activation='tanh', inner_activation='hard_sigmoid',
                                            W_regularizer=None, U_regularizer=None, b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                        prev_bck = LSTM(hidden_dim, return_sequences=True,go_backwards=True,stateful=False,
                                            init='glorot_uniform', inner_init='orthogonal',
                                            forget_bias_init='one', activation='tanh', inner_activation='hard_sigmoid',
                                            W_regularizer=None, U_regularizer=None, b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                    x = merge([prev_frw, prev_bck], mode='concat')
                                output_layer = TimeDistributed(Dense(3,activation='softmax'))(x)
                                model = Model(input=[input_layer],output=[output_layer])
                                model.compile(optimizer=optimizer,
                                                loss='sparse_categorical_crossentropy',
                                                metrics=['accuracy'],
                                                sample_weight_mode="temporal")

                                ########save architecture ######
                                arch_dir = './data/models/archs/'+short_title+'.yml'
                                yaml_string = model.to_yaml()
                                with open(arch_dir, 'wb') as output:
                                    pickle.dump(yaml_string, output, pickle.HIGHEST_PROTOCOL)
                                    print 'model saved!'
                                ##############

                                user_mode = 'train'
                                trans_mode = 'train'
                                data_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                 batch_size=batch_size,usr_ratio=80,class_weight=class_weight,lbl_pad_val = lbl_pad_val, pad_val = pad_val,
                                                 sub_sample=user_sample_size,epoch_size=epoch_limit,events_tbl=events_tbl)
                                                 # sub_sample=user_sample_size,epoch_size=samples_per_epoch)
                                ########validation data
                                print 'Generating Validation set!'
                                user_mode = 'test'
                                trans_mode = 'test'
                                val_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                 batch_size=batch_size_val,usr_ratio=80,class_weight=class_weight,lbl_pad_val = lbl_pad_val, pad_val = pad_val,
                                                 sub_sample=None,epoch_size=None,events_tbl=events_tbl)
                                validation_data = next(val_gen)
                                print '################GENERATED#######################'
                                ###############CALLBACKS
                                patience = 30
                                early_Stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0, mode='auto')

                                save_path = './data/models/'+table+'/'
                                var_name = '.{epoch:02d}-{val_loss:.5f}.hdf5'
                                checkpoint = keras.callbacks.ModelCheckpoint(save_path+short_title+var_name, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')

                                root_url = 'http://localhost:9000'
                                remote_log = keras.callbacks.RemoteMonitor(root=root_url)

                                # callbacks = [early_Stop,checkpoint]
                                callbacks = [early_Stop,checkpoint,remote_log]
                                callbacks = []
                                history = model.fit_generator(data_gen, samples_per_epoch, nb_epoch, verbose=1, callbacks=callbacks,validation_data=validation_data, nb_val_samples=None, class_weight=None, max_q_size=10000)

                                py.sign_in('bottydim', 'o1kuyms9zv')

                                auc_list = []
                                print '#########################TRAIN STATS################'
                                user_mode = 'train'
                                trans_mode = 'train'
                                val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                print '# samples',val_samples
                                plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"

                                data_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                 batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl)

                                eval_list  = eval_auc_generator(model, data_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                auc_val = eval_list[0]
                                clc_report = eval_list[1]
                                acc = eval_list[2]
                                print "AUC:",auc_val
                                print 'CLassification report'
                                print clc_report
                                print 'Accuracy'
                                print acc
                                auc_list.append(str(auc_val))
                                print '##################EVALUATION USERS#########################'

                                user_mode = 'test'
                                trans_mode = 'train'
                                val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                print '# samples',val_samples
                                plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"

                                eval_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                 batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl)

                                eval_list  = eval_auc_generator(model, eval_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                auc_val = eval_list[0]
                                clc_report = eval_list[1]
                                acc = eval_list[2]
                                print "AUC:",auc_val
                                print 'CLassification report'
                                print clc_report
                                print 'Accuracy'
                                print acc
                                auc_list.append(str(auc_val))
                                print '#####################################################'
                                print '##################EVALUATION Transactions#########################'

                                user_mode = 'train'
                                trans_mode = 'test'
                                val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                print '# samples',val_samples
                                plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"

                                eval_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                 batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl)

                                eval_list  = eval_auc_generator(model, eval_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                auc_val = eval_list[0]
                                clc_report = eval_list[1]
                                acc = eval_list[2]
                                print "AUC:",auc_val
                                print 'CLassification report'
                                print clc_report
                                print 'Accuracy'
                                print acc
                                auc_list.append(str(auc_val))
                                print '#####################################################'
                                print '##################EVALUATION Pure#########################'

                                user_mode = 'test'
                                trans_mode = 'test'
                                val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                print '# samples',val_samples
                                plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"

                                eval_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                 batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl)

                                eval_list  = eval_auc_generator(model, eval_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                auc_val = eval_list[0]
                                clc_report = eval_list[1]
                                acc = eval_list[2]
                                print "AUC:",auc_val
                                print 'CLassification report'
                                print clc_report
                                print 'Accuracy'
                                print acc
                                auc_list.append(str(auc_val))
                                print '#####################################################'
                                with io.open(rsl_file, 'a', encoding='utf-8') as file:
                                    auc_string = ','.join(auc_list)
                                    title_csv = title.replace('_',',')+','+str(history.history['acc'][-1])+','+str(history.history['loss'][-1])+','+str(auc_val)+','+str(acc)+','+auc_string+'\n'
                                    file.write(unicode(title_csv))
                                    print 'logged @ {file}'.format(file=rsl_file)
                                trim_point = -15
                                fig = {
                                    'data': [Scatter(
                                        x=history.epoch[trim_point:],
                                        y=history.history['loss'][trim_point:])],
                                    'layout': {'title': title}
                                    }
                                py.image.save_as(fig,filename='./results/figures/'+table+'/'+short_title+'_'+'LOSS'+'_'+add_info+".png")
                                trim_point = 0
                                fig = {
                                    'data': [Scatter(
                                        x=history.epoch[trim_point:],
                                        y=history.history['loss'][trim_point:])],
                                    'layout': {'title': title}
                                    }
                                py.image.save_as(fig,filename='./results/figures/'+table+'/'+short_title+'_'+'LOSS'+'_'+'FULL'+".png")

                                # iplot(fig,filename='figures/'+title,image='png')
                                # title = title.replace('Loss','Acc')
                                fig = {
                                    'data': [Scatter(
                                        x=history.epoch[trim_point:],
                                        y=history.history['acc'][trim_point:])],
                                    'layout': {'title': title}
                                    }
                                filename_val='./results/figures/'+table+'/'+short_title+'_'+'ACC'+'_'+add_info+".png"
                                py.image.save_as(fig,filename=filename_val)
                                print 'exported @',filename_val
                                fig = {
                                    'data': [Scatter(
                                        x=history.epoch[trim_point:],
                                        y=history.history['val_loss'][trim_point:])],
                                    'layout': {'title': title}
                                    }
                                py.image.save_as(fig,filename='./results/figures/'+table+'/'+short_title+'_'+'VAL LOSS'+'_'+add_info+".png")
                                print 'time taken: {time}'.format(time=days_hours_minutes_seconds(dt.datetime.now()-t_start))


"""
====================================================================
Normal and Shrinkage Linear Discriminant Analysis for classification
====================================================================

Shows how shrinkage improves classification.
"""

from __future__ import division

import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_blobs
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

n_train = 20  # samples for training
n_test = 200  # samples for testing
n_averages = 50  # how often to repeat classification
n_features_max = 75  # maximum number of features
step = 4  # step size for the calculation

def generate_data(n_samples, n_features):
    """Generate random blob-ish data with noisy features.

    This returns an array of input data with shape `(n_samples, n_features)`
    and an array of `n_samples` target labels.

    Only one feature contains discriminative information, the other features
    contain only noise.
    """
    X, y = make_blobs(n_samples=n_samples, n_features=1, centers=[[-2], [2]])

    # add non-discriminative features
    if n_features > 1:
        X = np.hstack([X, np.random.randn(n_samples, n_features - 1)])
    return X, y

acc_clf1, acc_clf2 = [], []
n_features_range = range(1, n_features_max + 1, step)
for n_features in n_features_range:
    score_clf1, score_clf2 = 0, 0
    for _ in range(n_averages):
        X, y = generate_data(n_train, n_features)

        clf1 = LinearDiscriminantAnalysis(solver='lsqr', shrinkage='auto').fit(X, y)
        clf2 = LinearDiscriminantAnalysis(solver='lsqr', shrinkage=None).fit(X, y)

        X, y = generate_data(n_test, n_features)
        score_clf1 += clf1.score(X, y)
        score_clf2 += clf2.score(X, y)

    acc_clf1.append(score_clf1 / n_averages)
    acc_clf2.append(score_clf2 / n_averages)

features_samples_ratio = np.array(n_features_range) / n_train

plt.plot(features_samples_ratio, acc_clf1, linewidth=2,
         label="Linear Discriminant Analysis with shrinkage", color='r')
plt.plot(features_samples_ratio, acc_clf2, linewidth=2,
         label="Linear Discriminant Analysis", color='g')

plt.xlabel('n_features / n_samples')
plt.ylabel('Classification accuracy')

plt.legend(loc=1, prop={'size': 12})
plt.suptitle('Linear Discriminant Analysis vs. \
shrinkage Linear Discriminant Analysis (1 discriminative feature)')
plt.show()


#!/usr/bin/env python
#
# Copyright 2015 Free Software Foundation, Inc.
#
# GNU Radio is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GNU Radio is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNU Radio; see the file COPYING.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street,
# Boston, MA 02110-1301, USA.
#

import numpy as np
import helper_functions as hf

def bec_channel(eta):
    '''
    binary erasure channel (BEC)
    for each y e Y
    W(y|0) * W(y|1) = 0 or W(y|0) = W(y|1)
    transistions are 1 -> 1 or 0 -> 0 or {0, 1} -> ? (erased symbol)
    '''
    # looks like BSC but should be interpreted differently.
    w = np.array((1 - eta, eta, 1 - eta), dtype=float)
    return w

def odd_rec(iwn):
    return iwn ** 2

def even_rec(iwn):
    return 2 * iwn - iwn ** 2

def calc_one_recursion(iw0):
    iw1 = np.zeros(2 * len(iw0))  # double values
    for i in range(len(iw0)):
        # careful indices screw you because paper is '1' based :(
        iw1[2 * i] = odd_rec(iw0[i])
        iw1[2 * i + 1] = even_rec(iw0[i])
    return iw1

def calculate_bec_channel_capacities_loop(initial_channel, block_power):
        # compare [0, Arikan] eq. 6
    iw = np.array([initial_channel, ], dtype=float)
    for i in range(block_power):
        iw = calc_one_recursion(iw)
    return iw

def calc_vector_capacities_one_recursion(iw0):
    degraded = odd_rec(iw0)
    upgraded = even_rec(iw0)
    iw1 = np.empty(2 * len(iw0), dtype=degraded.dtype)
    iw1[0::2] = degraded
    iw1[1::2] = upgraded
    return iw1

def calculate_bec_channel_capacities_vector(initial_channel, block_power):
    # compare [0, Arikan] eq. 6
    # this version is ~ 180 times faster than the loop version with 2**22 synthetic channels
    iw = np.array([initial_channel, ], dtype=float)
    for i in range(block_power):
        iw = calc_vector_capacities_one_recursion(iw)
    return iw

def calculate_bec_channel_capacities(eta, block_size):
    # compare [0, Arikan] eq. 6
    iw = 1 - eta  # holds for BEC as stated in paper
    lw = hf.power_of_2_int(block_size)
    return calculate_bec_channel_capacities_vector(iw, lw)

def calculate_z_parameters_one_recursion(z_params):
    z_next = np.empty(2 * z_params.size, dtype=z_params.dtype)
    z_sq = z_params ** 2
    z_low = 2 * z_params - z_sq
    z_next[0::2] = z_low
    z_next[1::2] = z_sq
    return z_next

def calculate_bec_channel_z_parameters(eta, block_size):
    # compare [0, Arikan] eq. 38
    block_power = hf.power_of_2_int(block_size)
    z_params = np.array([eta, ], dtype=float)
    for block_size in range(block_power):
        z_params = calculate_z_parameters_one_recursion(z_params)
    return z_params

def design_snr_to_bec_eta(design_snr):
    # minimum design snr = -1.5917 corresponds to BER = 0.5
    s = 10. ** (design_snr / 10.)
    return np.exp(-s)

def bhattacharyya_bounds(design_snr, block_size):
    '''
    Harish Vangala, Emanuele Viterbo, Yi Hong: 'A Comparative Study of Polar Code Constructions for the AWGN Channel', 2015
    In this paper it is called Bhattacharyya bounds channel construction and is abbreviated PCC-0
    Best design SNR for block_size = 2048, R = 0.5, is 0dB.
    Compare with Arikan: 'Channel Polarization: A Method for Constructing Capacity-Achieving Codes for Symmetric Binary-Input Memoryless Channels.
    Proposition 5. inequalities turn into equalities for BEC channel. Otherwise they represent an upper bound.
    Also compare [0, Arikan] eq. 6 and 38
    For BEC that translates to capacity(i) = 1 - bhattacharyya(i)
    :return Z-parameters in natural bit-order. Choose according to desired rate.
    '''
    eta = design_snr_to_bec_eta(design_snr)
    return calculate_bec_channel_z_parameters(eta, block_size)

def plot_channel_capacities(capacity, save_file=None):
    block_size = len(capacity)
    try:
        import matplotlib.pyplot as plt
        # FUN with matplotlib LaTeX fonts! http://matplotlib.org/users/usetex.html
        plt.rc('text', usetex=True)
        plt.rc('font', family='serif')
        plt.rc('figure', autolayout=True)
        plt.plot(capacity)
        plt.xlim([0, block_size])
        plt.ylim([-0.01, 1.01])
        plt.xlabel('synthetic channel number')
        plt.ylabel('channel capacity')
        # plt.title('BEC channel construction')
        plt.grid()
        plt.gcf().set_size_inches(plt.gcf().get_size_inches() * .5)
        if save_file:
            plt.savefig(save_file)
        plt.show()
    except ImportError:
        pass  # only plot in case matplotlib is installed

def plot_average_channel_distance(save_file=None):
    eta = 0.5 #  design_snr_to_bec_eta(-1.5917)
    powers = np.arange(4, 26)

    try:
        import matplotlib.pyplot as plt
        import matplotlib
        # FUN with matplotlib LaTeX fonts! http://matplotlib.org/users/usetex.html
        plt.rc('text', usetex=True)
        plt.rc('font', family='serif')
        plt.rc('figure', autolayout=True)

        dist = []
        medians = []
        initial_channel = 1 - eta
        for p in powers:
            bs = int(2 ** p)
            capacities = calculate_bec_channel_capacities(eta, bs)
            avg_capacity = np.repeat(initial_channel, len(capacities))
            averages = np.abs(capacities - avg_capacity)
            avg_distance = np.sum(averages) / float(len(capacities))
            dist.append(avg_distance)
            variance = np.std(averages)
            medians.append(variance)

        plt.errorbar(powers, dist, yerr=medians)
        plt.grid()
        plt.xlabel(r'block size $N$')
        plt.ylabel(r'$\frac{1}{N} \sum_i |I(W_N^{(i)}) - 0.5|$')

        axes = plt.axes()
        tick_values = np.array(axes.get_xticks().tolist())
        tick_labels = np.array(tick_values, dtype=int)
        tick_labels = ['$2^{' + str(i) + '}$' for i in tick_labels]
        plt.xticks(tick_values, tick_labels)
        plt.xlim((powers[0], powers[-1]))
        plt.ylim((0.2, 0.5001))
        plt.gcf().set_size_inches(plt.gcf().get_size_inches() * .5)
        if save_file:
            plt.savefig(save_file)
        plt.show()
    except ImportError:
        pass

def plot_capacity_histogram(design_snr, save_file=None):
    eta = design_snr_to_bec_eta(design_snr)
    # capacities = calculate_bec_channel_capacities(eta, block_size)
    try:
        import matplotlib.pyplot as plt
        # FUN with matplotlib LaTeX fonts! http://matplotlib.org/users/usetex.html
        plt.rc('text', usetex=True)
        plt.rc('font', family='serif')
        plt.rc('figure', autolayout=True)

        block_sizes = [32, 128, 512]
        for b in block_sizes:
            capacities = calculate_bec_channel_capacities(eta, b)
            w = 1. / float(len(capacities))
            weights = [w, ] * b
            plt.hist(capacities, bins=b, weights=weights, range=(0.95, 1.0))
        plt.grid()
        plt.xlabel('synthetic channel capacity')
        plt.ylabel('normalized item count')
        print(plt.gcf().get_size_inches())
        plt.gcf().set_size_inches(plt.gcf().get_size_inches() * .5)
        if save_file:
            plt.savefig(save_file)
        plt.show()
    except ImportError:
        pass

def main():
    print 'channel construction main'
    n = 11
    block_size = int(2 ** n)
    design_snr = -1.59
    eta = design_snr_to_bec_eta(design_snr)
    # print(calculate_bec_channel_z_parameters(eta, block_size))
    # capacity = calculate_bec_channel_capacities(eta, block_size)
    # plot_average_channel_distance()
    calculate_bec_channel_z_parameters(eta, block_size)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as _plt

class DraggableLegend:
    def __init__(self, legend):
        self.legend = legend
        self.gotLegend = False
        legend.figure.canvas.mpl_connect('motion_notify_event', self.on_motion)
        legend.figure.canvas.mpl_connect('pick_event', self.on_picker)
        legend.figure.canvas.mpl_connect('button_release_event', self.on_release)
        legend.set_picker(self.my_legend_picker)

    #----------------------------------------------------#
    # Connected event handlers

    def on_motion(self, event):
        if self.gotLegend:
            dx = event.x - self.mouse_x
            dy = event.y - self.mouse_y
            loc_in_canvas = self.legend_x + dx, self.legend_y + dy
            loc_in_norm_axes = self.legend.parent.transAxes.inverted().transform_point(loc_in_canvas)
            self.legend._loc = tuple(loc_in_norm_axes)
            self.legend.figure.canvas.draw()

    def my_legend_picker(self, legend, event):
        return self.legend.legendPatch.contains(event)

    def on_picker(self, event):
        if event.artist == self.legend:
            # left-click
            if event.mouseevent.button == 1:
                self._move_legend(event)

            # mouse button pressed
            if event.mouseevent.button == 2:
                pass

            # right-click
            if event.mouseevent.button == 3:
                self._hideLegend()

            # mouse up
            if event.mouseevent.button == 'up':
                self._scaleUpLegendFont()

            # mouse down
            if event.mouseevent.button == 'down':
                self._scaleDownLegendFont()

    def on_release(self, event):
        if self.gotLegend:
            self.gotLegend = False

    #----------------------------------------------------#
    # Utility functions

    def _move_legend(self,event):
            bbox = self.legend.get_window_extent()
            self.mouse_x = event.mouseevent.x
            self.mouse_y = event.mouseevent.y
            self.legend_x = bbox.xmin
            self.legend_y = bbox.ymin
            self.gotLegend = 1

    def _scaleUpLegendFont(self,size_step=4):
        size = self.legend.get_texts()[0].get_fontsize()
        size += size_step
        _plt.setp(self.legend.get_texts(), fontsize=size) #legend 'list' fontsize
        self.legend.figure.canvas.draw()

    def _scaleDownLegendFont(self,size_step=4):
        size = self.legend.get_texts()[0].get_fontsize()
        size -= size_step
        _plt.setp(self.legend.get_texts(), fontsize=size) #legend 'list' fontsize
        self.legend.figure.canvas.draw()

    def _hideLegend(self):
        if self.legend.get_visible():
            self.legend.set_visible(False)
        else:
            self.legend.set_visible(True)
        self.legend.figure.canvas.draw()

figure = _plt.figure()
ax = figure.add_subplot(111)
scatter = ax.scatter(np.random.randn(100), np.random.randn(100), label='hi')
legend = ax.legend()
legend = DraggableLegend(legend)
_plt.show()


# -*- coding: utf-8 -*-
"""
Created on Fri Feb 12 01:01:50 2010

Author: josef-pktd

latest result
-------------
all are very close
garch0 has different parameterization of constant
ordering of parameters is different

seed 2780185
h.shape (2000,)
Optimization terminated successfully.
         Current function value: 2093.813397
         Iterations: 387
         Function evaluations: 676
ggres.params [-0.6146253   0.1914537   0.01039355  0.78802188]
Optimization terminated successfully.
         Current function value: 2093.972953
         Iterations: 201
         Function evaluations: 372
ggres0.params [-0.61537527  0.19635128  4.00706058]
Warning: Desired error not necessarily achieveddue to precision loss
         Current function value: 2093.972953
         Iterations: 51
         Function evaluations: 551
         Gradient evaluations: 110
ggres0.params [-0.61537855  0.19635265  4.00694669]
Optimization terminated successfully.
         Current function value: 2093.751420
         Iterations: 103
         Function evaluations: 187
[ 0.78671519  0.19692222  0.61457171]
-2093.75141963

Final Estimate:
 LLH:  2093.750    norm LLH:  2.093750
    omega    alpha1     beta1
0.7867438 0.1970437 0.6145467

long run variance comparison
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
R
>>> 0.7867438/(1- 0.1970437- 0.6145467)
4.1757097302897526
Garch (gjr) asymetric, longrun var ?
>>> 1/(1-0.6146253 - 0.1914537 - 0.01039355) * 0.78802188
4.2937548579245242
>>> 1/(1-0.6146253 - 0.1914537 + 0.01039355) * 0.78802188
3.8569053452140345
Garch0
>>> (1-0.61537855 - 0.19635265) * 4.00694669
0.7543830449902722
>>> errgjr4.var() #for different random seed
4.0924199964716106

todo: add code and verify, check for longer lagpolys

"""

import numpy as np
from numpy.testing import assert_almost_equal

import matplotlib.pyplot as plt
import numdifftools as ndt

import scikits.statsmodels.api as sm
from scikits.statsmodels.sandbox import tsa
from scikits.statsmodels.sandbox.tsa.garch import *  # local import

nobs = 1000
examples = ['garch', 'rpyfit']
if 'garch' in examples:
    err,h = generate_kindofgarch(nobs, [1.0, -0.95], [1.0,  0.1], mu=0.5)
    plt.figure()
    plt.subplot(211)
    plt.plot(err)
    plt.subplot(212)
    plt.plot(h)
    #plt.show()

    seed = 3842774 #91234  #8837708
    seed = np.random.randint(9999999)
    print 'seed', seed
    np.random.seed(seed)
    ar1 = -0.9
    err,h = generate_garch(nobs, [1.0, ar1], [1.0,  0.50], mu=0.0,scale=0.1)
#    plt.figure()
#    plt.subplot(211)
#    plt.plot(err)
#    plt.subplot(212)
#    plt.plot(h)
#    plt.figure()
#    plt.subplot(211)
#    plt.plot(err[-400:])
#    plt.subplot(212)
#    plt.plot(h[-400:])
    #plt.show()
    garchplot(err, h)
    garchplot(err[-400:], h[-400:])

    np.random.seed(seed)
    errgjr,hgjr, etax = generate_gjrgarch(nobs, [1.0, ar1],
                                [[1,0],[0.5,0]], mu=0.0,scale=0.1)
    garchplot(errgjr[:nobs], hgjr[:nobs], 'GJR-GARCH(1,1) Simulation - symmetric')
    garchplot(errgjr[-400:nobs], hgjr[-400:nobs], 'GJR-GARCH(1,1) Simulation - symmetric')

    np.random.seed(seed)
    errgjr2,hgjr2, etax = generate_gjrgarch(nobs, [1.0, ar1],
                                [[1,0],[0.1,0.9]], mu=0.0,scale=0.1)
    garchplot(errgjr2[:nobs], hgjr2[:nobs], 'GJR-GARCH(1,1) Simulation')
    garchplot(errgjr2[-400:nobs], hgjr2[-400:nobs], 'GJR-GARCH(1,1) Simulation')

    np.random.seed(seed)
    errgjr3,hgjr3, etax3 = generate_gjrgarch(nobs, [1.0, ar1],
                        [[1,0],[0.1,0.9],[0.1,0.9],[0.1,0.9]], mu=0.0,scale=0.1)
    garchplot(errgjr3[:nobs], hgjr3[:nobs], 'GJR-GARCH(1,3) Simulation')
    garchplot(errgjr3[-400:nobs], hgjr3[-400:nobs], 'GJR-GARCH(1,3) Simulation')

    np.random.seed(seed)
    errgjr4,hgjr4, etax4 = generate_gjrgarch(nobs, [1.0, ar1],
                        [[1., 1,0],[0, 0.1,0.9],[0, 0.1,0.9],[0, 0.1,0.9]],
                        mu=0.0,scale=0.1)
    garchplot(errgjr4[:nobs], hgjr4[:nobs], 'GJR-GARCH(1,3) Simulation')
    garchplot(errgjr4[-400:nobs], hgjr4[-400:nobs], 'GJR-GARCH(1,3) Simulation')

    varinno = np.zeros(100)
    varinno[0] = 1.
    errgjr5,hgjr5, etax5 = generate_gjrgarch(100, [1.0, -0.],
                        [[1., 1,0],[0, 0.1,0.8],[0, 0.05,0.7],[0, 0.01,0.6]],
                        mu=0.0,scale=0.1, varinnovation=varinno)
    garchplot(errgjr5[:20], hgjr5[:20], 'GJR-GARCH(1,3) Simulation')
    #garchplot(errgjr4[-400:nobs], hgjr4[-400:nobs], 'GJR-GARCH(1,3) Simulation')

#plt.show()
seed = np.random.randint(9999999)  # 9188410
print 'seed', seed

x = np.arange(20).reshape(10,2)
x3 = np.column_stack((np.ones((x.shape[0],1)),x))
y, inp = miso_lfilter([1., 0],np.array([[-2.0,3,1],[0.0,0.0,0]]),x3)

nobs = 1000
warmup = 1000
np.random.seed(seed)
ar = [1.0, -0.7]#7, -0.16, -0.1]
#ma = [[1., 1, 0],[0, 0.6,0.1],[0, 0.1,0.1],[0, 0.1,0.1]]
ma = [[1., 0, 0],[0, 0.8,0.0]] #,[0, 0.9,0.0]]
#    errgjr4,hgjr4, etax4 = generate_gjrgarch(warmup+nobs, [1.0, -0.99],
#                        [[1., 1, 0],[0, 0.6,0.1],[0, 0.1,0.1],[0, 0.1,0.1]],
#                        mu=0.2, scale=0.25)

errgjr4,hgjr4, etax4 = generate_gjrgarch(warmup+nobs, ar, ma,
                     mu=0.4, scale=1.01)
errgjr4,hgjr4, etax4 = errgjr4[warmup:], hgjr4[warmup:], etax4[warmup:]
garchplot(errgjr4[:nobs], hgjr4[:nobs], 'GJR-GARCH(1,3) Simulation - DGP')
ggmod = Garch(errgjr4-errgjr4.mean())#hgjr4[:nobs])#-hgjr4.mean()) #errgjr4)
ggmod.nar = 1
ggmod.nma = 1
ggmod._start_params = np.array([-0.6, 0.1, 0.2, 0.0])
ggres = ggmod.fit(start_params=np.array([-0.6, 0.1, 0.2, 0.0]), maxiter=1000)
print 'ggres.params', ggres.params
garchplot(ggmod.errorsest, ggmod.h, title='Garch estimated')

ggmod0 = Garch0(errgjr4-errgjr4.mean())#hgjr4[:nobs])#-hgjr4.mean()) #errgjr4)
ggmod0.nar = 1
ggmod.nma = 1
start_params = np.array([-0.6, 0.2, 0.1])
ggmod0._start_params = start_params #np.array([-0.6, 0.1, 0.2, 0.0])
ggres0 = ggmod0.fit(start_params=start_params, maxiter=2000)
print 'ggres0.params', ggres0.params

ggmod0 = Garch0(errgjr4-errgjr4.mean())#hgjr4[:nobs])#-hgjr4.mean()) #errgjr4)
ggmod0.nar = 1
ggmod.nma = 1
start_params = np.array([-0.6, 0.2, 0.1])
ggmod0._start_params = start_params #np.array([-0.6, 0.1, 0.2, 0.0])
ggres0 = ggmod0.fit(start_params=start_params, method='bfgs', maxiter=2000)
print 'ggres0.params', ggres0.params

g11res = optimize.fmin(lambda params: -loglike_GARCH11(params, errgjr4-errgjr4.mean())[0], [0.93, 0.9, 0.2])
print g11res
llf = loglike_GARCH11(g11res, errgjr4-errgjr4.mean())
print llf[0]

if 'rpyfit' in examples:
    from rpy import r
    r.library('fGarch')
    f = r.formula('~garch(1, 1)')
    fit = r.garchFit(f, data = errgjr4-errgjr4.mean(), include_mean=False)

if 'rpysim' in examples:
    from rpy import r
    f = r.formula('~garch(1, 1)')
    #fit = r.garchFit(f, data = errgjr4)
    x = r.garchSim( n = 500)
    print 'R acf', tsa.acf(np.power(x,2))[:15]
    arma3 = Arma(np.power(x,2))
    arma3res = arma3.fit(start_params=[-0.2,0.1,0.5],maxiter=5000)
    print arma3res.params
    arma3b = Arma(np.power(x,2))
    arma3bres = arma3b.fit(start_params=[-0.2,0.1,0.5],maxiter=5000, method='bfgs')
    print arma3bres.params

    xr = r.garchSim( n = 100)

    x = np.asarray(xr)
    ggmod = Garch(x-x.mean())
    ggmod.nar = 1
    ggmod.nma = 1
    ggmod._start_params = np.array([-0.6, 0.1, 0.2, 0.0])
    ggres = ggmod.fit(start_params=np.array([-0.6, 0.1, 0.2, 0.0]), maxiter=1000)
    print 'ggres.params', ggres.params

    g11res = optimize.fmin(lambda params: -loglike_GARCH11(params, x-x.mean())[0], [0.6, 0.6, 0.2])
    print g11res
    llf = loglike_GARCH11(g11res, x-x.mean())
    print llf[0]

    garchplot(ggmod.errorsest, ggmod.h, title='Garch estimated')
    fit = r.garchFit(f, data = x-x.mean(), include_mean=False, trace=False)
    print r.summary(fit)

'''based on R default simulation
model = list(omega = 1e-06, alpha = 0.1, beta = 0.8)
nobs = 1000
(with nobs=500, gjrgarch doesn't do well

>>> ggres = ggmod.fit(start_params=np.array([-0.6, 0.1, 0.2, 0.0]), maxiter=1000)
Optimization terminated successfully.
         Current function value: -448.861335
         Iterations: 385
         Function evaluations: 690
>>> print 'ggres.params', ggres.params
ggres.params [ -7.75090330e-01   1.57714749e-01  -9.60223930e-02   8.76021411e-07]
rearranged
8.76021411e-07 1.57714749e-01(-9.60223930e-02) 7.75090330e-01

>>> print g11res
[  2.97459808e-06   7.83128600e-01   2.41110860e-01]
>>> llf = loglike_GARCH11(g11res, x-x.mean())
>>> print llf[0]
442.603541936

Log Likelihood:
 -448.9376    normalized:  -4.489376
      omega       alpha1        beta1
1.01632e-06  1.02802e-01  7.57537e-01
'''

''' the following is for errgjr4-errgjr4.mean()
ggres.params [-0.54510407  0.22723132  0.06482633  0.82325803]
Final Estimate:
 LLH:  2065.56    norm LLH:  2.06556
        mu      omega     alpha1      beta1
0.07229732 0.83069480 0.26313883 0.53986167

ggres.params [-0.50779163  0.2236606   0.00700036  1.154832
Final Estimate:
 LLH:  2116.084    norm LLH:  2.116084
           mu         omega        alpha1         beta1
-4.759227e-17  1.145404e+00  2.288348e-01  5.085949e-01

run3
DGP
0.4/??    0.8      0.7
gjrgarch:
ggres.params [-0.45196579  0.2569641   0.02201904  1.11942636]
rearranged
const/omega  ma1/alpha1             ar1/beta1
1.11942636 0.2569641(+0.02201904) 0.45196579
g11:
[ 1.10262688  0.26680468  0.45724957]
-2055.73912687
R:
Final Estimate:
 LLH:  2055.738    norm LLH:  2.055738
           mu         omega        alpha1         beta1
-1.665226e-17  1.102396e+00  2.668712e-01  4.573224e-01
fit = r.garchFit(f, data = errgjr4-errgjr4.mean())
rpy.RPy_RException: Error in solve.default(fit$hessian) :
  Lapack routine dgesv: system is exactly singular

run4
DGP:
mu=0.4, scale=1.01
ma = [[1., 0, 0],[0, 0.8,0.0]], ar = [1.0, -0.7]
maybe something wrong with simulation

gjrgarch
ggres.params [-0.50554663  0.24449867 -0.00521004  1.00796791]
rearranged
1.00796791   0.24449867(-0.00521004)   0.50554663
garch11:
[ 1.01258264  0.24149155  0.50479994]
-2056.3877404
R include_constant=False
Final Estimate:
 LLH:  2056.397    norm LLH:  2.056397
    omega    alpha1     beta1
1.0123560 0.2409589 0.5049154
'''

erro,ho, etaxo = generate_gjrgarch(20, ar, ma, mu=0.04, scale=0.01,
                  varinnovation = np.ones(20))

if 'sp500' in examples:
    import tabular as tb
    import scikits.timeseries as ts

    a = tb.loadSV(r'C:\Josef\work-oth\gspc_table.csv')

    s = ts.time_series(a[0]['Close'][::-1],
                dates=ts.date_array(a[0]['Date'][::-1],freq="D"))

    sp500 = a[0]['Close'][::-1]
    sp500r = np.diff(np.log(sp500))

#plt.show()


# Import Modules as needed
import numpy as np
#import seaborn as sn
import pandas as pd
from pylab import *
from mylocal_functions import *

# ======== T2 MSME============= #
# Make list of all T2.txt files
T2_list = get_ipython().getoutput('ls ../Study_03_CBA/*T2.txt')

# Allocate variables needed for analysis
T2DF=pd.DataFrame()
TR=np.linspace(.012,.012*12,12)
# Fit T2 and construct dataframe
for names in T2_list:
    #Convert txt file to array
    YDataMatrix=txt_2_array(names)
    #Estimate T2
    T2time=fitT2(TR,YDataMatrix)
    #convert to data frame
    df_T2=pd.DataFrame(T2time.T,columns=["Infected","Healthy_R","St_Inf","Healthy_L"])
    #df_T2=pd.DataFrame(T2time.T,columns=["ROI-1","ROI-2","ROI-3","ROI-4"])
    df_info=name_2_df(names)
    df_final=pd.concat([df_T2,df_info], axis=1)
    T2DF=T2DF.append(df_final,ignore_index=True)

# Plot T2 Density ROIs 1 and 2
#T2DF[T2DF.Slice==1].iloc[:,:4].plot.density(); title("Slice 01"); xlim((0.025,.15))
#T2DF[T2DF.Slice==2].iloc[:,:4].plot.density(); title("Slice 02"); xlim((0.025,.15))
#T2DF[T2DF.Slice==3].iloc[:,:4].plot.density(); title("Slice 03"); xlim((0.025,.15))
#T2DF[T2DF.Slice==4].iloc[:,:4].plot.density(); title("Slice 04"); xlim((0.025,.15))
#T2DF[T2DF.Slice==5].iloc[:,:4].plot.density(); title("Slice 05"); xlim((0.025,.15))

# ======== CEST============= #
# Make list of all T2.txt files
CEST_list=get_ipython().getoutput('ls ../Study_03_CBA/*CEST.txt')
CEST_DF=pd.DataFrame()
Z=np.zeros((4,110))

def normalize_data(DataMatrix):
    rows,cols = DataMatrix.shape
    newData = np.zeros_like(DataMatrix)
    for row in range(rows):
        newData[row,:]=DataMatrix[row,:]/DataMatrix[row,8]
    return newData

for names in CEST_list:
    #Convert txt file to array
    D=txt_2_array(names);
    Zn=normalize_data(D.T)
    Z=np.concatenate((Z,Zn))

Z=Z[4::,9::]

# define offsets in ppm
a1=np.linspace(-55,-50,9)
ppm=np.linspace(-8,8,101)
full_ppm = np.concatenate((a1, ppm))

# fit CEST data.
y=Z[12,:]
p=fit_L2_scale(ppm,y)
Yhat=Lscale(ppm,p[0],p[1],p[2],p[3],p[4],p[5],p[6]);

plt.figure(figsize=(10,6))
plt.plot(ppm,y,'o',label='Signal');
plt.plot(ppm,1-Yhat,'-',label='Fit');
plt.legend()

## ======   BUILD CEST Predictors ======== #####
CEST_predictors=np.zeros_like(Z)
rows,cols = CEST_predictors.shape
Tissue_Class=np.zeros((4,rows))

for i in range(rows):
    p=fit_L2_scale(ppm,Z[i,:])
    CEST_predictors[i,:]=Lscale(ppm,p[0],p[1],p[2],p[3],p[4],p[5],p[6]);

Tissue_Class=np.zeros((64,1))

for i in range(4):
    Tissue_Class[i::4]=i

CEST_Dataframe=pd.DataFrame(CEST_predictors)
CEST_Dataframe["Tissue_Class"]=Tissue_Class

pd.DataFrame.to_csv(CEST_Dataframe,"CEST_infections.csv",header=True,index=False)


"""Gaussian processes regression. """

# Authors: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#
# License: BSD 3 clause

import warnings
from operator import itemgetter

import numpy as np
from scipy.linalg import cholesky, cho_solve, solve_triangular
from scipy.optimize import fmin_l_bfgs_b

from sklearn.base import BaseEstimator, RegressorMixin, clone
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
from sklearn.utils import check_random_state
from sklearn.utils.validation import check_X_y, check_array

class GaussianProcessRegressor(BaseEstimator, RegressorMixin):
    """Gaussian process regression (GPR).

    The implementation is based on Algorithm 2.1 of Gaussian Processes
    for Machine Learning (GPML) by Rasmussen and Williams.

    In addition to standard sklearn estimator API, GaussianProcessRegressor:

       * allows prediction without prior fitting (based on the GP prior)
       * provides an additional method sample_y(X), which evaluates samples
         drawn from the GPR (prior or posterior) at given inputs
       * exposes a method log_marginal_likelihood(theta), which can be used
         externally for other ways of selecting hyperparameters, e.g., via
         Markov chain Monte Carlo.

    Parameters
    ----------
    kernel : kernel object
        The kernel specifying the covariance function of the GP. If None is
        passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that
        the kernel's hyperparameters are optimized during fitting.

    alpha : float or array-like, optional (default: 1e-10)
        Value added to the diagonal of the kernel matrix during fitting.
        Larger values correspond to increased noise level in the observations
        and reduce potential numerical issue during fitting. If an array is
        passed, it must have the same number of entries as the data used for
        fitting and is used as datapoint-dependent noise level. Note that this
        is equivalent to adding a WhiteKernel with c=alpha. Allowing to specify
        the noise level directly as a parameter is mainly for convenience and
        for consistency with Ridge.

    optimizer : string or callable, optional (default: "fmin_l_bfgs_b")
        Can either be one of the internally supported optimizers for optimizing
        the kernel's parameters, specified by a string, or an externally
        defined optimizer passed as a callable. If a callable is passed, it
        must have the signature::

            def optimizer(obj_func, initial_theta, bounds):
                # * 'obj_func' is the objective function to be maximized, which
                #   takes the hyperparameters theta as parameter and an
                #   optional flag eval_gradient, which determines if the
                #   gradient is returned additionally to the function value
                # * 'initial_theta': the initial value for theta, which can be
                #   used by local optimizers
                # * 'bounds': the bounds on the values of theta
                ....
                # Returned are the best found hyperparameters theta and
                # the corresponding value of the target function.
                return theta_opt, func_min

        Per default, the 'fmin_l_bfgs_b' algorithm from scipy.optimize
        is used. If None is passed, the kernel's parameters are kept fixed.
        Available internal optimizers are::

            'fmin_l_bfgs_b'

    n_restarts_optimizer: int, optional (default: 0)
        The number of restarts of the optimizer for finding the kernel's
        parameters which maximize the log-marginal likelihood. The first run
        of the optimizer is performed from the kernel's initial parameters,
        the remaining ones (if any) from thetas sampled log-uniform randomly
        from the space of allowed theta-values. If greater than 0, all bounds
        must be finite. Note that n_restarts_optimizer == 0 implies that one
        run is performed.

    normalize_y: boolean, optional (default: False)
        Whether the target values y are normalized, i.e., the mean of the
        observed target values become zero. This parameter should be set to
        True if the target values' mean is expected to differ considerable from
        zero. When enabled, the normalization effectively modifies the GP's
        prior based on the data, which contradicts the likelihood principle;
        normalization is thus disabled per default.

    copy_X_train : bool, optional (default: True)
        If True, a persistent copy of the training data is stored in the
        object. Otherwise, just a reference to the training data is stored,
        which might cause predictions to change if the data is modified
        externally.

    random_state : integer or numpy.RandomState, optional
        The generator used to initialize the centers. If an integer is
        given, it fixes the seed. Defaults to the global numpy random
        number generator.

    Attributes
    ----------
    X_train_ : array-like, shape = (n_samples, n_features)
        Feature values in training data (also required for prediction)

    y_train_: array-like, shape = (n_samples, [n_output_dims])
        Target values in training data (also required for prediction)

    kernel_: kernel object
        The kernel used for prediction. The structure of the kernel is the
        same as the one passed as parameter but with optimized hyperparameters

    L_: array-like, shape = (n_samples, n_samples)
        Lower-triangular Cholesky decomposition of the kernel in ``X_train_``

    alpha_: array-like, shape = (n_samples,)
        Dual coefficients of training data points in kernel space

    log_marginal_likelihood_value_: float
        The log-marginal-likelihood of ``self.kernel_.theta``
    """
    def __init__(self, kernel=None, alpha=1e-10,
                 optimizer="fmin_l_bfgs_b", n_restarts_optimizer=0,
                 normalize_y=False, copy_X_train=True, random_state=None):
        self.kernel = kernel
        self.alpha = alpha
        self.optimizer = optimizer
        self.n_restarts_optimizer = n_restarts_optimizer
        self.normalize_y = normalize_y
        self.copy_X_train = copy_X_train
        self.random_state = random_state

    def fit(self, X, y):
        """Fit Gaussian process regression model

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Training data

        y : array-like, shape = (n_samples, [n_output_dims])
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        if self.kernel is None:  # Use an RBF kernel as default
            self.kernel_ = C(1.0, constant_value_bounds="fixed") \
                * RBF(1.0, length_scale_bounds="fixed")
        else:
            self.kernel_ = clone(self.kernel)

        self.rng = check_random_state(self.random_state)

        X, y = check_X_y(X, y, multi_output=True, y_numeric=True)

        # Normalize target value
        if self.normalize_y:
            self.y_train_mean = np.mean(y, axis=0)
            # demean y
            y = y - self.y_train_mean
        else:
            self.y_train_mean = np.zeros(1)

        if np.iterable(self.alpha) \
           and self.alpha.shape[0] != y.shape[0]:
            if self.alpha.shape[0] == 1:
                self.alpha = self.alpha[0]
            else:
                raise ValueError("alpha must be a scalar or an array"
                                 " with same number of entries as y.(%d != %d)"
                                 % (self.alpha.shape[0], y.shape[0]))

        self.X_train_ = np.copy(X) if self.copy_X_train else X
        self.y_train_ = np.copy(y) if self.copy_X_train else y

        if self.optimizer is not None and self.kernel_.n_dims > 0:
            # Choose hyperparameters based on maximizing the log-marginal
            # likelihood (potentially starting from several initial values)
            def obj_func(theta, eval_gradient=True):
                if eval_gradient:
                    lml, grad = self.log_marginal_likelihood(
                        theta, eval_gradient=True)
                    return -lml, -grad
                else:
                    return -self.log_marginal_likelihood(theta)

            # First optimize starting from theta specified in kernel
            optima = [(self._constrained_optimization(obj_func,
                                                      self.kernel_.theta,
                                                      self.kernel_.bounds))]

            # Additional runs are performed from log-uniform chosen initial
            # theta
            if self.n_restarts_optimizer > 0:
                if not np.isfinite(self.kernel_.bounds).all():
                    raise ValueError(
                        "Multiple optimizer restarts (n_restarts_optimizer>0) "
                        "requires that all bounds are finite.")
                bounds = self.kernel_.bounds
                for iteration in range(self.n_restarts_optimizer):
                    theta_initial = \
                        self.rng.uniform(bounds[:, 0], bounds[:, 1])
                    optima.append(
                        self._constrained_optimization(obj_func, theta_initial,
                                                       bounds))
            # Select result from run with minimal (negative) log-marginal
            # likelihood
            lml_values = list(map(itemgetter(1), optima))
            self.kernel_.theta = optima[np.argmin(lml_values)][0]
            self.log_marginal_likelihood_value_ = -np.min(lml_values)
        else:
            self.log_marginal_likelihood_value_ = \
                self.log_marginal_likelihood(self.kernel_.theta)

        # Precompute quantities required for predictions which are independent
        # of actual query points
        K = self.kernel_(self.X_train_)
        K[np.diag_indices_from(K)] += self.alpha
        self.L_ = cholesky(K, lower=True)  # Line 2
        self.alpha_ = cho_solve((self.L_, True), self.y_train_)  # Line 3

        return self

    def predict(self, X, return_std=False, return_cov=False):
        """Predict using the Gaussian process regression model

        We can also predict based on an unfitted model by using the GP prior.
        In addition to the mean of the predictive distribution, also its
        standard deviation (return_std=True) or covariance (return_cov=True).
        Note that at most one of the two can be requested.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Query points where the GP is evaluated

        return_std : bool, default: False
            If True, the standard-deviation of the predictive distribution at
            the query points is returned along with the mean.

        return_cov : bool, default: False
            If True, the covariance of the joint predictive distribution at
            the query points is returned along with the mean

        Returns
        -------
        y_mean : array, shape = (n_samples, [n_output_dims])
            Mean of predictive distribution a query points

        y_std : array, shape = (n_samples,), optional
            Standard deviation of predictive distribution at query points.
            Only returned when return_std is True.

        y_cov : array, shape = (n_samples, n_samples), optional
            Covariance of joint predictive distribution a query points.
            Only returned when return_cov is True.
        """
        if return_std and return_cov:
            raise RuntimeError(
                "Not returning standard deviation of predictions when "
                "returning full covariance.")

        X = check_array(X)

        if not hasattr(self, "X_train_"):  # Unfitted;predict based on GP prior
            y_mean = np.zeros(X.shape[0])
            if return_cov:
                y_cov = self.kernel(X)
                return y_mean, y_cov
            elif return_std:
                y_var = self.kernel.diag(X)
                return y_mean, np.sqrt(y_var)
            else:
                return y_mean
        else:  # Predict based on GP posterior
            K_trans = self.kernel_(X, self.X_train_)
            y_mean = K_trans.dot(self.alpha_)  # Line 4 (y_mean = f_star)
            y_mean = self.y_train_mean + y_mean  # undo normal.
            if return_cov:
                v = cho_solve((self.L_, True), K_trans.T)  # Line 5
                y_cov = self.kernel_(X) - K_trans.dot(v)  # Line 6
                return y_mean, y_cov
            elif return_std:
                # compute inverse K_inv of K based on its Cholesky
                # decomposition L and its inverse L_inv
                L_inv = solve_triangular(self.L_.T, np.eye(self.L_.shape[0]))
                K_inv = L_inv.dot(L_inv.T)
                # Compute variance of predictive distribution
                y_var = self.kernel_.diag(X)
                y_var -= np.einsum("ki,kj,ij->k", K_trans, K_trans, K_inv)

                # Check if any of the variances is negative because of
                # numerical issues. If yes: set the variance to 0.
                y_var_negative = y_var < 0
                if np.any(y_var_negative):
                    warnings.warn("Predicted variances smaller than 0. "
                                  "Setting those variances to 0.")
                    y_var[y_var_negative] = 0.0
                return y_mean, np.sqrt(y_var)
            else:
                return y_mean

    def sample_y(self, X, n_samples=1, random_state=0):
        """Draw samples from Gaussian process and evaluate at X.

        Parameters
        ----------
        X : array-like, shape = (n_samples_X, n_features)
            Query points where the GP samples are evaluated

        n_samples : int, default: 1
            The number of samples drawn from the Gaussian process

        random_state: RandomState or an int seed (0 by default)
            A random number generator instance

        Returns
        -------
        y_samples : array, shape = (n_samples_X, [n_output_dims], n_samples)
            Values of n_samples samples drawn from Gaussian process and
            evaluated at query points.
        """
        rng = check_random_state(random_state)

        y_mean, y_cov = self.predict(X, return_cov=True)
        if y_mean.ndim == 1:
            y_samples = rng.multivariate_normal(y_mean, y_cov, n_samples).T
        else:
            y_samples = \
                [rng.multivariate_normal(y_mean[:, i], y_cov,
                                         n_samples).T[:, np.newaxis]
                 for i in range(y_mean.shape[1])]
            y_samples = np.hstack(y_samples)
        return y_samples

    def log_marginal_likelihood(self, theta=None, eval_gradient=False):
        """Returns log-marginal likelihood of theta for training data.

        Parameters
        ----------
        theta : array-like, shape = (n_kernel_params,) or None
            Kernel hyperparameters for which the log-marginal likelihood is
            evaluated. If None, the precomputed log_marginal_likelihood
            of ``self.kernel_.theta`` is returned.

        eval_gradient : bool, default: False
            If True, the gradient of the log-marginal likelihood with respect
            to the kernel hyperparameters at position theta is returned
            additionally. If True, theta must not be None.

        Returns
        -------
        log_likelihood : float
            Log-marginal likelihood of theta for training data.

        log_likelihood_gradient : array, shape = (n_kernel_params,), optional
            Gradient of the log-marginal likelihood with respect to the kernel
            hyperparameters at position theta.
            Only returned when eval_gradient is True.
        """
        if theta is None:
            if eval_gradient:
                raise ValueError(
                    "Gradient can only be evaluated for theta!=None")
            return self.log_marginal_likelihood_value_

        kernel = self.kernel_.clone_with_theta(theta)

        if eval_gradient:
            K, K_gradient = kernel(self.X_train_, eval_gradient=True)
        else:
            K = kernel(self.X_train_)

        K[np.diag_indices_from(K)] += self.alpha
        try:
            L = cholesky(K, lower=True)  # Line 2
        except np.linalg.LinAlgError:
            return (-np.inf, np.zeros_like(theta)) \
                if eval_gradient else -np.inf

        # Support multi-dimensional output of self.y_train_
        y_train = self.y_train_
        if y_train.ndim == 1:
            y_train = y_train[:, np.newaxis]

        alpha = cho_solve((L, True), y_train)  # Line 3

        # Compute log-likelihood (compare line 7)
        log_likelihood_dims = -0.5 * np.einsum("ik,ik->k", y_train, alpha)
        log_likelihood_dims -= np.log(np.diag(L)).sum()
        log_likelihood_dims -= K.shape[0] / 2 * np.log(2 * np.pi)
        log_likelihood = log_likelihood_dims.sum(-1)  # sum over dimensions

        if eval_gradient:  # compare Equation 5.9 from GPML
            tmp = np.einsum("ik,jk->ijk", alpha, alpha)  # k: output-dimension
            tmp -= cho_solve((L, True), np.eye(K.shape[0]))[:, :, np.newaxis]
            # Compute "0.5 * trace(tmp.dot(K_gradient))" without
            # constructing the full matrix tmp.dot(K_gradient) since only
            # its diagonal is required
            log_likelihood_gradient_dims = \
                0.5 * np.einsum("ijl,ijk->kl", tmp, K_gradient)
            log_likelihood_gradient = log_likelihood_gradient_dims.sum(-1)

        if eval_gradient:
            return log_likelihood, log_likelihood_gradient
        else:
            return log_likelihood

    def _constrained_optimization(self, obj_func, initial_theta, bounds):
        if self.optimizer == "fmin_l_bfgs_b":
            theta_opt, func_min, convergence_dict = \
                fmin_l_bfgs_b(obj_func, initial_theta, bounds=bounds)
            if convergence_dict["warnflag"] != 0:
                warnings.warn("fmin_l_bfgs_b terminated abnormally with the "
                              " state: %s" % convergence_dict)
        elif callable(self.optimizer):
            theta_opt, func_min = \
                self.optimizer(obj_func, initial_theta, bounds=bounds)
        else:
            raise ValueError("Unknown optimizer %s." % self.optimizer)

        return theta_opt, func_min


import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import hamming, hanning, triang, blackmanharris, resample
import math
import sys, os, time

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../software/models/'))

import stft as STFT
import utilFunctions as UF
import harmonicModel as HM

(fs, x) = UF.wavread(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../sounds/flute-A4.wav'))
w = np.blackman(551)
N = 1024
t = -100
nH = 40
minf0 = 420
maxf0 = 460
f0et = 5
maxnpeaksTwm = 5
minSineDur = .1
harmDevSlope = 0.01
Ns = 512
H = Ns/4

mX, pX = STFT.stftAnal(x, fs, w, N, H)
hfreq, hmag, hphase = HM.harmonicModelAnal(x, fs, w, N, H, t, nH, minf0, maxf0, f0et, harmDevSlope, minSineDur)
xr = UF.sineSubtraction(x, Ns, H, hfreq, hmag, hphase, fs)
mXr, pXr = STFT.stftAnal(xr, fs, hamming(Ns), Ns, H)

maxplotfreq = 5000.0
plt.figure(1, figsize=(9, 7))

plt.subplot(221)
numFrames = int(mX[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
binFreq = fs*np.arange(N*maxplotfreq/fs)/N
plt.pcolormesh(frmTime, binFreq, np.transpose(mX[:,:N*maxplotfreq/fs+1]))
plt.autoscale(tight=True)

harms = hfreq*np.less(hfreq,maxplotfreq)
harms[harms==0] = np.nan
numFrames = int(harms[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
plt.plot(frmTime, harms, color='k', ms=3, alpha=1)
plt.autoscale(tight=True)
plt.title('mX + harmonics (flute-A4.wav)')

plt.subplot(222)
numFrames = int(mX[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
binFreq = fs*np.arange(N*maxplotfreq/fs)/N
plt.pcolormesh(frmTime, binFreq, np.transpose(np.diff(pX[:,:N*maxplotfreq/fs+1],axis=1)))
plt.autoscale(tight=True)

harms = hfreq*np.less(hfreq,maxplotfreq)
harms[harms==0] = np.nan
numFrames = int(harms[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
plt.plot(frmTime, harms, color='k', ms=3, alpha=1)
plt.autoscale(tight=True)
plt.title('pX + harmonics')

plt.subplot(223)
numFrames = int(mXr[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
binFreq = fs*np.arange(Ns*maxplotfreq/fs)/Ns
plt.pcolormesh(frmTime, binFreq, np.transpose(mXr[:,:Ns*maxplotfreq/fs+1]))
plt.autoscale(tight=True)
plt.title('mXr')

plt.subplot(224)
numFrames = int(pXr[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
binFreq = fs*np.arange(Ns*maxplotfreq/fs)/Ns
plt.pcolormesh(frmTime, binFreq, np.transpose(np.diff(pXr[:,:Ns*maxplotfreq/fs+1],axis=1)))
plt.autoscale(tight=True)
plt.title('pXr')

plt.tight_layout()
plt.savefig('hprModelAnal-flute.png')
UF.wavwrite(5*xr, fs, 'flute-residual.wav')
plt.show()


"""
==============================================
Plot randomly generated multilabel dataset
==============================================

This illustrates the `datasets.make_multilabel_classification` dataset
generator. Each sample consists of counts of two features (up to 50 in
total), which are differently distributed in each of two classes.

Points are labeled as follows, where Y means the class is present:

    =====  =====  =====  ======
      1      2      3    Color
    =====  =====  =====  ======
      Y      N      N    Red
      N      Y      N    Blue
      N      N      Y    Yellow
      Y      Y      N    Purple
      Y      N      Y    Orange
      Y      Y      N    Green
      Y      Y      Y    Brown
    =====  =====  =====  ======

A star marks the expected sample for each class; its size reflects the
probability of selecting that class label.

The left and right examples highlight the ``n_labels`` parameter:
more of the samples in the right plot have 2 or 3 labels.

Note that this two-dimensional example is very degenerate:
generally the number of features would be much greater than the
"document length", while here we have much larger documents than vocabulary.
Similarly, with ``n_classes > n_features``, it is much less likely that a
feature distinguishes a particular class.
"""

from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import make_multilabel_classification as make_ml_clf

print(__doc__)

COLORS = np.array(['!',
                   '#FF3333',  # red
                   '#0198E1',  # blue
                   '#BF5FFF',  # purple
                   '#FCD116',  # yellow
                   '#FF7216',  # orange
                   '#4DBD33',  # green
                   '#87421F'   # brown
                   ])

# Use same random seed for multiple calls to make_multilabel_classification to
# ensure same distributions
RANDOM_SEED = np.random.randint(2 ** 10)

def plot_2d(ax, n_labels=1, n_classes=3, length=50):
    X, Y, p_c, p_w_c = make_ml_clf(n_samples=150, n_features=2,
                                   n_classes=n_classes, n_labels=n_labels,
                                   length=length, allow_unlabeled=False,
                                   return_distributions=True,
                                   random_state=RANDOM_SEED)

    ax.scatter(X[:, 0], X[:, 1], color=COLORS.take((Y * [1, 2, 4]
                                                    ).sum(axis=1)),
               marker='.')
    ax.scatter(p_w_c[0] * length, p_w_c[1] * length,
               marker='*', linewidth=.5, edgecolor='black',
               s=20 + 1500 * p_c ** 2,
               color=COLORS.take([1, 2, 4]))
    ax.set_xlabel('Feature 0 count')
    return p_c, p_w_c

_, (ax1, ax2) = plt.subplots(1, 2, sharex='row', sharey='row', figsize=(8, 4))
plt.subplots_adjust(bottom=.15)

p_c, p_w_c = plot_2d(ax1, n_labels=1)
ax1.set_title('n_labels=1, length=50')
ax1.set_ylabel('Feature 1 count')

plot_2d(ax2, n_labels=3)
ax2.set_title('n_labels=3, length=50')
ax2.set_xlim(left=0, auto=True)
ax2.set_ylim(bottom=0, auto=True)

plt.show()

print('The data was generated from (random_state=%d):' % RANDOM_SEED)
print('Class', 'P(C)', 'P(w0|C)', 'P(w1|C)', sep='\t')
for k, p, p_w in zip(['red', 'blue', 'yellow'], p_c, p_w_c.T):
    print('%s\t%0.2f\t%0.2f\t%0.2f' % (k, p, p_w[0], p_w[1]))


# -*- coding: utf-8 -*-

"""
Tests the TextReader class in parsers.pyx, which
is integral to the C engine in parsers.py
"""

from pandas.compat import StringIO, BytesIO, map
from pandas import compat

import os
import sys
import nose

from numpy import nan
import numpy as np

from pandas import DataFrame
from pandas.io.parsers import (read_csv, TextFileReader)
from pandas.util.testing import assert_frame_equal

import pandas.util.testing as tm

from pandas.parser import TextReader
import pandas.parser as parser

class TestTextReader(tm.TestCase):

    def setUp(self):
        self.dirpath = tm.get_data_path()
        self.csv1 = os.path.join(self.dirpath, 'test1.csv')
        self.csv2 = os.path.join(self.dirpath, 'test2.csv')
        self.xls1 = os.path.join(self.dirpath, 'test.xls')

    def test_file_handle(self):
        try:
            f = open(self.csv1, 'rb')
            reader = TextReader(f)
            result = reader.read()  # noqa
        finally:
            f.close()

    def test_string_filename(self):
        reader = TextReader(self.csv1, header=None)
        reader.read()

    def test_file_handle_mmap(self):
        try:
            f = open(self.csv1, 'rb')
            reader = TextReader(f, memory_map=True, header=None)
            reader.read()
        finally:
            f.close()

    def test_StringIO(self):
        with open(self.csv1, 'rb') as f:
            text = f.read()
        src = BytesIO(text)
        reader = TextReader(src, header=None)
        reader.read()

    def test_string_factorize(self):
        # should this be optional?
        data = 'a\nb\na\nb\na'
        reader = TextReader(StringIO(data), header=None)
        result = reader.read()
        self.assertEqual(len(set(map(id, result[0]))), 2)

    def test_skipinitialspace(self):
        data = ('a,   b\n'
                'a,   b\n'
                'a,   b\n'
                'a,   b')

        reader = TextReader(StringIO(data), skipinitialspace=True,
                            header=None)
        result = reader.read()

        self.assert_numpy_array_equal(result[0],
                                      np.array(['a', 'a', 'a', 'a'],
                                               dtype=np.object_))
        self.assert_numpy_array_equal(result[1],
                                      np.array(['b', 'b', 'b', 'b'],
                                               dtype=np.object_))

    def test_parse_booleans(self):
        data = 'True\nFalse\nTrue\nTrue'

        reader = TextReader(StringIO(data), header=None)
        result = reader.read()

        self.assertEqual(result[0].dtype, np.bool_)

    def test_delimit_whitespace(self):
        data = 'a  b\na\t\t "b"\n"a"\t \t b'

        reader = TextReader(StringIO(data), delim_whitespace=True,
                            header=None)
        result = reader.read()

        self.assert_numpy_array_equal(result[0], np.array(['a', 'a', 'a'],
                                                          dtype=np.object_))
        self.assert_numpy_array_equal(result[1], np.array(['b', 'b', 'b'],
                                                          dtype=np.object_))

    def test_embedded_newline(self):
        data = 'a\n"hello\nthere"\nthis'

        reader = TextReader(StringIO(data), header=None)
        result = reader.read()

        expected = np.array(['a', 'hello\nthere', 'this'], dtype=np.object_)
        self.assert_numpy_array_equal(result[0], expected)

    def test_euro_decimal(self):
        data = '12345,67\n345,678'

        reader = TextReader(StringIO(data), delimiter=':',
                            decimal=',', header=None)
        result = reader.read()

        expected = np.array([12345.67, 345.678])
        tm.assert_almost_equal(result[0], expected)

    def test_integer_thousands(self):
        data = '123,456\n12,500'

        reader = TextReader(StringIO(data), delimiter=':',
                            thousands=',', header=None)
        result = reader.read()

        expected = np.array([123456, 12500], dtype=np.int64)
        tm.assert_almost_equal(result[0], expected)

    def test_integer_thousands_alt(self):
        data = '123.456\n12.500'

        reader = TextFileReader(StringIO(data), delimiter=':',
                                thousands='.', header=None)
        result = reader.read()

        expected = DataFrame([123456, 12500])
        tm.assert_frame_equal(result, expected)

    def test_skip_bad_lines(self):
        # too many lines, see #2430 for why
        data = ('a:b:c\n'
                'd:e:f\n'
                'g:h:i\n'
                'j:k:l:m\n'
                'l:m:n\n'
                'o:p:q:r')

        reader = TextReader(StringIO(data), delimiter=':',
                            header=None)
        self.assertRaises(parser.CParserError, reader.read)

        reader = TextReader(StringIO(data), delimiter=':',
                            header=None,
                            error_bad_lines=False,
                            warn_bad_lines=False)
        result = reader.read()
        expected = {0: ['a', 'd', 'g', 'l'],
                    1: ['b', 'e', 'h', 'm'],
                    2: ['c', 'f', 'i', 'n']}
        assert_array_dicts_equal(result, expected)

        stderr = sys.stderr
        sys.stderr = StringIO()
        try:
            reader = TextReader(StringIO(data), delimiter=':',
                                header=None,
                                error_bad_lines=False,
                                warn_bad_lines=True)
            reader.read()
            val = sys.stderr.getvalue()
            self.assertTrue('Skipping line 4' in val)
            self.assertTrue('Skipping line 6' in val)
        finally:
            sys.stderr = stderr

    def test_header_not_enough_lines(self):
        data = ('skip this\n'
                'skip this\n'
                'a,b,c\n'
                '1,2,3\n'
                '4,5,6')

        reader = TextReader(StringIO(data), delimiter=',', header=2)
        header = reader.header
        expected = [['a', 'b', 'c']]
        self.assertEqual(header, expected)

        recs = reader.read()
        expected = {0: [1, 4], 1: [2, 5], 2: [3, 6]}
        assert_array_dicts_equal(expected, recs)

        # not enough rows
        self.assertRaises(parser.CParserError, TextReader, StringIO(data),
                          delimiter=',', header=5, as_recarray=True)

    def test_header_not_enough_lines_as_recarray(self):
        data = ('skip this\n'
                'skip this\n'
                'a,b,c\n'
                '1,2,3\n'
                '4,5,6')

        reader = TextReader(StringIO(data), delimiter=',', header=2,
                            as_recarray=True)
        header = reader.header
        expected = [['a', 'b', 'c']]
        self.assertEqual(header, expected)

        recs = reader.read()
        expected = {'a': [1, 4], 'b': [2, 5], 'c': [3, 6]}
        assert_array_dicts_equal(expected, recs)

        # not enough rows
        self.assertRaises(parser.CParserError, TextReader, StringIO(data),
                          delimiter=',', header=5, as_recarray=True)

    def test_escapechar(self):
        data = ('\\"hello world\"\n'
                '\\"hello world\"\n'
                '\\"hello world\"')

        reader = TextReader(StringIO(data), delimiter=',', header=None,
                            escapechar='\\')
        result = reader.read()
        expected = {0: ['"hello world"'] * 3}
        assert_array_dicts_equal(result, expected)

    def test_eof_has_eol(self):
        # handling of new line at EOF
        pass

    def test_na_substitution(self):
        pass

    def test_numpy_string_dtype(self):
        data = """\
a,1
aa,2
aaa,3
aaaa,4
aaaaa,5"""

        def _make_reader(**kwds):
            return TextReader(StringIO(data), delimiter=',', header=None,
                              **kwds)

        reader = _make_reader(dtype='S5,i4')
        result = reader.read()

        self.assertEqual(result[0].dtype, 'S5')

        ex_values = np.array(['a', 'aa', 'aaa', 'aaaa', 'aaaaa'], dtype='S5')
        self.assertTrue((result[0] == ex_values).all())
        self.assertEqual(result[1].dtype, 'i4')

        reader = _make_reader(dtype='S4')
        result = reader.read()
        self.assertEqual(result[0].dtype, 'S4')
        ex_values = np.array(['a', 'aa', 'aaa', 'aaaa', 'aaaa'], dtype='S4')
        self.assertTrue((result[0] == ex_values).all())
        self.assertEqual(result[1].dtype, 'S4')

    def test_numpy_string_dtype_as_recarray(self):
        data = """\
a,1
aa,2
aaa,3
aaaa,4
aaaaa,5"""

        def _make_reader(**kwds):
            return TextReader(StringIO(data), delimiter=',', header=None,
                              **kwds)

        reader = _make_reader(dtype='S4', as_recarray=True)
        result = reader.read()
        self.assertEqual(result['0'].dtype, 'S4')
        ex_values = np.array(['a', 'aa', 'aaa', 'aaaa', 'aaaa'], dtype='S4')
        self.assertTrue((result['0'] == ex_values).all())
        self.assertEqual(result['1'].dtype, 'S4')

    def test_pass_dtype(self):
        data = """\
one,two
1,a
2,b
3,c
4,d"""

        def _make_reader(**kwds):
            return TextReader(StringIO(data), delimiter=',', **kwds)

        reader = _make_reader(dtype={'one': 'u1', 1: 'S1'})
        result = reader.read()
        self.assertEqual(result[0].dtype, 'u1')
        self.assertEqual(result[1].dtype, 'S1')

        reader = _make_reader(dtype={'one': np.uint8, 1: object})
        result = reader.read()
        self.assertEqual(result[0].dtype, 'u1')
        self.assertEqual(result[1].dtype, 'O')

        reader = _make_reader(dtype={'one': np.dtype('u1'),
                                     1: np.dtype('O')})
        result = reader.read()
        self.assertEqual(result[0].dtype, 'u1')
        self.assertEqual(result[1].dtype, 'O')

    def test_usecols(self):
        data = """\
a,b,c
1,2,3
4,5,6
7,8,9
10,11,12"""

        def _make_reader(**kwds):
            return TextReader(StringIO(data), delimiter=',', **kwds)

        reader = _make_reader(usecols=(1, 2))
        result = reader.read()

        exp = _make_reader().read()
        self.assertEqual(len(result), 2)
        self.assertTrue((result[1] == exp[1]).all())
        self.assertTrue((result[2] == exp[2]).all())

    def test_cr_delimited(self):
        def _test(text, **kwargs):
            nice_text = text.replace('\r', '\r\n')
            result = TextReader(StringIO(text), **kwargs).read()
            expected = TextReader(StringIO(nice_text), **kwargs).read()
            assert_array_dicts_equal(result, expected)

        data = 'a,b,c\r1,2,3\r4,5,6\r7,8,9\r10,11,12'
        _test(data, delimiter=',')

        data = 'a  b  c\r1  2  3\r4  5  6\r7  8  9\r10  11  12'
        _test(data, delim_whitespace=True)

        data = 'a,b,c\r1,2,3\r4,5,6\r,88,9\r10,11,12'
        _test(data, delimiter=',')

        sample = ('A,B,C,D,E,F,G,H,I,J,K,L,M,N,O\r'
                  'AAAAA,BBBBB,0,0,0,0,0,0,0,0,0,0,0,0,0\r'
                  ',BBBBB,0,0,0,0,0,0,0,0,0,0,0,0,0')
        _test(sample, delimiter=',')

        data = 'A  B  C\r  2  3\r4  5  6'
        _test(data, delim_whitespace=True)

        data = 'A B C\r2 3\r4 5 6'
        _test(data, delim_whitespace=True)

    def test_empty_field_eof(self):
        data = 'a,b,c\n1,2,3\n4,,'

        result = TextReader(StringIO(data), delimiter=',').read()

        expected = {0: np.array([1, 4]),
                    1: np.array(['2', ''], dtype=object),
                    2: np.array(['3', ''], dtype=object)}
        assert_array_dicts_equal(result, expected)

        # GH5664
        a = DataFrame([['b'], [nan]], columns=['a'], index=['a', 'c'])
        b = DataFrame([[1, 1, 1, 0], [1, 1, 1, 0]],
                      columns=list('abcd'),
                      index=[1, 1])
        c = DataFrame([[1, 2, 3, 4], [6, nan, nan, nan],
                       [8, 9, 10, 11], [13, 14, nan, nan]],
                      columns=list('abcd'),
                      index=[0, 5, 7, 12])

        for _ in range(100):
            df = read_csv(StringIO('a,b\nc\n'), skiprows=0,
                          names=['a'], engine='c')
            assert_frame_equal(df, a)

            df = read_csv(StringIO('1,1,1,1,0\n' * 2 + '\n' * 2),
                          names=list("abcd"), engine='c')
            assert_frame_equal(df, b)

            df = read_csv(StringIO('0,1,2,3,4\n5,6\n7,8,9,10,11\n12,13,14'),
                          names=list('abcd'), engine='c')
            assert_frame_equal(df, c)

def assert_array_dicts_equal(left, right):
    for k, v in compat.iteritems(left):
        assert(np.array_equal(v, right[k]))

if __name__ == '__main__':
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)