#  Copyright 2016 Intel Corporation
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from brainiak.fcma.classifier import Classifier
from brainiak.fcma.preprocessing import prepare_fcma_data
from brainiak.io import dataset
from sklearn import svm
#from sklearn.linear_model import LogisticRegression
import sys
import logging
import numpy as np
from scipy.spatial.distance import hamming
from sklearn import model_selection
#from sklearn.externals import joblib

format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
# if want to output log to a file instead of outputting log to the console,
# replace "stream=sys.stdout" with "filename='fcma.log'"
logging.basicConfig(level=logging.INFO, format=format, stream=sys.stdout)
logger = logging.getLogger(__name__)

def example_of_aggregating_sim_matrix(raw_data, labels, num_subjects, num_epochs_per_subj):
    # aggregate the kernel matrix to save memory
    svm_clf = svm.SVC(kernel='precomputed', shrinking=False, C=1)
    clf = Classifier(svm_clf, num_processed_voxels=1000, epochs_per_subj=num_epochs_per_subj)
    rearranged_data = raw_data[num_epochs_per_subj:] + raw_data[0:num_epochs_per_subj]
    rearranged_labels = labels[num_epochs_per_subj:] + labels[0:num_epochs_per_subj]
    clf.fit(list(zip(rearranged_data, rearranged_data)), rearranged_labels,
            num_training_samples=num_epochs_per_subj*(num_subjects-1))
    predict = clf.predict()
    print(predict)
    print(clf.decision_function())
    test_labels = labels[0:num_epochs_per_subj]
    incorrect_predict = hamming(predict, np.asanyarray(test_labels)) * num_epochs_per_subj
    logger.info(
        'when aggregating the similarity matrix to save memory, '
        'the accuracy is %d / %d = %.2f' %
        (num_epochs_per_subj-incorrect_predict, num_epochs_per_subj,
         (num_epochs_per_subj-incorrect_predict) * 1.0 / num_epochs_per_subj)
    )
    # when the kernel matrix is computed in portion, the test data is already in
    print(clf.score(None, test_labels))

def example_of_cross_validation_with_detailed_info(raw_data, labels, num_subjects, num_epochs_per_subj):
    # no shrinking, set C=1
    svm_clf = svm.SVC(kernel='precomputed', shrinking=False, C=1)
    #logit_clf = LogisticRegression()
    clf = Classifier(svm_clf, epochs_per_subj=num_epochs_per_subj)
    # doing leave-one-subject-out cross validation
    for i in range(num_subjects):
        leave_start = i * num_epochs_per_subj
        leave_end = (i+1) * num_epochs_per_subj
        training_data = raw_data[0:leave_start] + raw_data[leave_end:]
        test_data = raw_data[leave_start:leave_end]
        training_labels = labels[0:leave_start] + labels[leave_end:]
        test_labels = labels[leave_start:leave_end]
        clf.fit(list(zip(training_data, training_data)), training_labels)
        # joblib can be used for saving and loading models
        #joblib.dump(clf, 'model/logistic.pkl')
        #clf = joblib.load('model/svm.pkl')
        predict = clf.predict(list(zip(test_data, test_data)))
        print(predict)
        print(clf.decision_function(list(zip(test_data, test_data))))
        incorrect_predict = hamming(predict, np.asanyarray(test_labels)) * num_epochs_per_subj
        logger.info(
            'when leaving subject %d out for testing, the accuracy is %d / %d = %.2f' %
            (i, num_epochs_per_subj-incorrect_predict, num_epochs_per_subj,
             (num_epochs_per_subj-incorrect_predict) * 1.0 / num_epochs_per_subj)
        )
        print(clf.score(list(zip(test_data, test_data)), test_labels))

def example_of_cross_validation_using_model_selection(raw_data, labels, num_subjects, num_epochs_per_subj):
    # NOTE: this method does not work for sklearn.svm.SVC with precomputed kernel
    # when the kernel matrix is computed in portions; also, this method only works
    # for self-correlation, i.e. correlation between the same data matrix.

    # no shrinking, set C=1
    svm_clf = svm.SVC(kernel='precomputed', shrinking=False, C=1)
    #logit_clf = LogisticRegression()
    clf = Classifier(svm_clf, epochs_per_subj=num_epochs_per_subj)
    # doing leave-one-subject-out cross validation
    # no shuffling in cv
    skf = model_selection.StratifiedKFold(n_splits=num_subjects,
                                          shuffle=False)
    scores = model_selection.cross_val_score(clf, list(zip(raw_data, raw_data)),
                                             y=labels,
                                             cv=skf)
    print(scores)
    logger.info(
        'the overall cross validation accuracy is %.2f' %
        np.mean(scores)
    )

def example_of_correlating_two_components(raw_data, raw_data2, labels, num_subjects, num_epochs_per_subj):
    # aggregate the kernel matrix to save memory
    svm_clf = svm.SVC(kernel='precomputed', shrinking=False, C=1)
    clf = Classifier(svm_clf, epochs_per_subj=num_epochs_per_subj)
    num_training_samples=num_epochs_per_subj*(num_subjects-1)
    clf.fit(list(zip(raw_data[0:num_training_samples], raw_data2[0:num_training_samples])),
            labels[0:num_training_samples])
    X = list(zip(raw_data[num_training_samples:], raw_data2[num_training_samples:]))
    predict = clf.predict(X)
    print(predict)
    print(clf.decision_function(X))
    test_labels = labels[num_training_samples:]
    incorrect_predict = hamming(predict, np.asanyarray(test_labels)) * num_epochs_per_subj
    logger.info(
        'when aggregating the similarity matrix to save memory, '
        'the accuracy is %d / %d = %.2f' %
        (num_epochs_per_subj-incorrect_predict, num_epochs_per_subj,
         (num_epochs_per_subj-incorrect_predict) * 1.0 / num_epochs_per_subj)
    )
    # when the kernel matrix is computed in portion, the test data is already in
    print(clf.score(X, test_labels))

def example_of_correlating_two_components_aggregating_sim_matrix(raw_data, raw_data2, labels,
                                                                 num_subjects, num_epochs_per_subj):
    # aggregate the kernel matrix to save memory
    svm_clf = svm.SVC(kernel='precomputed', shrinking=False, C=1)
    clf = Classifier(svm_clf, num_processed_voxels=1000, epochs_per_subj=num_epochs_per_subj)
    num_training_samples=num_epochs_per_subj*(num_subjects-1)
    clf.fit(list(zip(raw_data, raw_data2)), labels,
            num_training_samples=num_training_samples)
    predict = clf.predict()
    print(predict)
    print(clf.decision_function())
    test_labels = labels[num_training_samples:]
    incorrect_predict = hamming(predict, np.asanyarray(test_labels)) * num_epochs_per_subj
    logger.info(
        'when aggregating the similarity matrix to save memory, '
        'the accuracy is %d / %d = %.2f' %
        (num_epochs_per_subj-incorrect_predict, num_epochs_per_subj,
         (num_epochs_per_subj-incorrect_predict) * 1.0 / num_epochs_per_subj)
    )
    # when the kernel matrix is computed in portion, the test data is already in
    print(clf.score(None, test_labels))

# python3 classification.py face_scene bet.nii.gz face_scene/prefrontal_top_mask.nii.gz face_scene/fs_epoch_labels.npy
if __name__ == '__main__':
    if len(sys.argv) != 5:
        logger.error('the number of input argument is not correct')
        sys.exit(1)

    data_dir = sys.argv[1]
    extension = sys.argv[2]
    mask_file = sys.argv[3]
    epoch_file = sys.argv[4]

    epoch_list = np.load(epoch_file)
    num_subjects = len(epoch_list)
    num_epochs_per_subj = epoch_list[0].shape[1]

    images = dataset.load_images_from_dir(data_dir, extension)
    mask = dataset.load_boolean_mask(mask_file)
    conditions = dataset.load_labels(epoch_file)
    raw_data, _, labels = prepare_fcma_data(images, conditions, mask)

    example_of_aggregating_sim_matrix(raw_data, labels, num_subjects, num_epochs_per_subj)

    example_of_cross_validation_with_detailed_info(raw_data, labels, num_subjects, num_epochs_per_subj)

    example_of_cross_validation_using_model_selection(raw_data, labels, num_subjects, num_epochs_per_subj)

    # test of two different components for correlation computation
    # images = dataset.load_images_from_dir(data_dir, extension)
    # mask2 = dataset.load_boolean_mask('face_scene/visual_top_mask.nii.gz')
    # raw_data, raw_data2, labels = prepare_fcma_data(images, conditions, mask,
    #                                                 mask2)
    #example_of_correlating_two_components(raw_data, raw_data2, labels, num_subjects, num_epochs_per_subj)
    #example_of_correlating_two_components_aggregating_sim_matrix(raw_data, raw_data2, labels, num_subjects, num_epochs_per_subj)


# Linear and nonlinear control of startup of a CSTR.
import mpctools as mpc
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import gridspec
import time

# Define some parameters and then the CSTR model.
Nx = 3
Nu = 2
Nd = 1
# Ny = Nx
Delta = .25
# eps = 1e-6 # Use this as a small number.

T0 = 350
c0 = 1
r = .219
k0 = 7.2e10
E = 8750
U = 54.94
rho = 1000
Cp = .239
dH = -5e4

def ode(x,u,d):
    # Grab the states, controls, and disturbance. We would like to write
    #
    # [c, T, h] = x[0:Nx]
    # [Tc, F] = u[0:Nu]
    # [F0] = d[0:Nd]
    #
    # but this doesn't work in Casadi 3.0. So, we're stuck with the following:
    c = x[0]
    T = x[1]
    h = x[2]
    Tc = u[0]
    F = u[1]
    F0 = d[0]

    # Now create the ODE.
    rate = k0*c*np.exp(-E/T)

    dxdt = np.array([
        F0*(c0 - c)/(np.pi*r**2*h) - rate,
        F0*(T0 - T)/(np.pi*r**2*h)
            - dH/(rho*Cp)*rate
            + 2*U/(r*rho*Cp)*(Tc - T),
        (F0 - F)/(np.pi*r**2)
    ])
    return dxdt

# Turn into casadi function and simulator.
ode_casadi = mpc.getCasadiFunc(ode,[Nx,Nu,Nd],["x","u","d"],funcname="ode")
ode_rk4_casadi = mpc.getCasadiFunc(ode,[Nx,Nu,Nd],["x","u","d"],
                                   funcname="ode_rk4",rk4=False,Delta=Delta)
cstr = mpc.DiscreteSimulator(ode, Delta, [Nx,Nu,Nd], ["x","u","d"])

# Steady-state values.
cs = .878
Ts = 324.5
hs = .659
Fs = .1
Tcs = 300
F0s = .1

# Update the steady-state values a few times to make sure they don't move.
for i in range(10):
    [cs,Ts,hs] = cstr.sim([cs,Ts,hs],[Tcs,Fs],[F0s]).tolist()
xs = np.array([cs,Ts,hs])
us = np.array([Tcs,Fs])
ds = np.array([F0s])

# Now get a linearization at this steady state.
#ss = mpc.util.getLinearizedModel(ode_casadi, [xs,us,ds], ["A","B","Bp"], Delta)
#A = ss["A"]
#B = ss["B"]
#Bp = ss["Bp"]
#C = np.eye(Nx)

# Weighting matrices for controller.
Q = .5*np.diag(xs**-2)
R = 2*np.diag(us**-2)

# model_casadi = mpc.getCasadiFunc(ode,[Nx,Nu,Nd],["x","u","d"],funcname="cstr")

#[K, Pi] = mpc.util.dlqr(A,B,Q,R)

# Define casadi functions.
Fnonlinear = ode_rk4_casadi

# def measurement(x,d):
#    return x
# h = mpc.getCasadiFunc(measurement,[Nx,Nd],["x","d"],funcname="h")

#def linmodel(x,u,d):
#    Ax = mpc.mtimes(A,x-xs) + xs
#    Bu = mpc.mtimes(B,u-us)
#    Bpd = mpc.mtimes(Bp,d-ds)
#    return Ax + Bu + Bpd
#Flinear = mpc.getCasadiFunc(linmodel,[Nx,Nu,Nd],["x","u","d"],funcname="F")

def stagecost(x,u,xsp,usp,Q,R):
    # Return deviation variables.
    dx = x - xsp
    du = u - usp
    # Calculate stage cost.
    return mpc.mtimes(dx.T,Q,dx) + mpc.mtimes(du.T,R,du)
largs = ["x","u","x_sp","u_sp","Q","R"]
l = mpc.getCasadiFunc(stagecost,[Nx,Nu,Nx,Nu,(Nx,Nx),(Nu,Nu)],largs,
                      funcname="l")

def costtogo(x,xsp):
    # Deviation variables.
    dx = x - xsp

    # Calculate cost to go.
    return mpc.mtimes(dx.T,10*Q,dx)
Pf = mpc.getCasadiFunc(costtogo,[Nx,Nx],["x","s_xp"],funcname="Pf")

# First see what happens if we try to start up the reactor under no control.
Nsim = 100
x0 = np.array([.05*cs,.75*Ts,.5*hs])
xcl = {}
ucl = {}
xcl["uncont"] = np.zeros((Nsim+1,Nx))
xcl["uncont"][0,:] = x0
ucl["uncont"] = np.tile(us,(Nsim,1))
for t in range(Nsim):
    xcl["uncont"][t+1,:] = cstr.sim(xcl["uncont"][t,:],ucl["uncont"][t,:],ds)

# Build a solver for the linear and nonlinear models.
Nt = 15
sp = {"x" : np.tile(xs, (Nt+1,1)), "u" : np.tile(us, (Nt,1))}
#xguesslin = np.zeros((Nt+1,Nx))
#xguesslin[0,:] = x0
#for t in range(Nt):
#    xguesslin[t+1,:] = A.dot(xguesslin[t,:] - xs) + xs
#guesslin = {"x" : xguesslin, "u" : np.tile(us,(Nt,1))}
guessnonlin = sp.copy()

# Control bounds.
umax = np.array([.05*Tcs,.15*Fs])
dumax = .2*umax # Maximum for rate-of-change.
bounds = dict(uub=[us + umax],ulb=[us - umax])
ub = {"u" : np.tile(us + umax, (Nt,1)), "Du" : np.tile(dumax, (Nt,1))}
lb = {"u" : np.tile(us - umax, (Nt,1)), "Du" : np.tile(-dumax, (Nt,1))}

N = {"x":Nx, "u":Nu, "p":Nd, "t":Nt, "c":3}
p = np.tile(ds, (Nt,1)) # Parameters for system.
nmpc_commonargs = {
    "N" : N,
    "Delta": Delta,
    "x0" : x0,
    "lb" : lb,
    "ub" : ub,
    "p" : p,
    "verbosity" : 0,
    "Pf" : Pf,
    "l" : l,
    "sp" : sp,
    "uprev" : us,
    "funcargs" : {"l" : largs},
    "extrapar" : {"Q" : Q, "R" : R}, # In case we want to tune online.
}
solvers = {}
# solvers["lmpc"] = mpc.nmpc(f=Flinear,guess=guesslin,**nmpc_commonargs)
solvers["nmpc"] = mpc.nmpc(f=Fnonlinear,guess=guessnonlin,**nmpc_commonargs)

# Also build steady-state target finders.
contVars = [0,2]
#sstarg_commonargs = {
#    "N" : N,
#    "lb" : {"u" : np.tile(us - umax, (1,1))},
#    "ub" : {"u" : np.tile(us + umax, (1,1))},
#    "verbosity" : 0,
##     "h" : h,
#    "p" : np.array([ds]),
#}
#sstargs = {}
# sstargs["lmpc"] = mpc.sstarg(f=Flinear,**sstarg_commonargs)
# sstargs["nmpc"] = mpc.sstarg(f=Fnonlinear,**sstarg_commonargs)

# Now simulate the process under control.
tcl = {}

for method in solvers.keys():
    xcl[method] = np.zeros((Nsim+1,Nx))
    xcl[method][0,:] = x0
    tcl[method] = np.zeros((Nsim+1,1))
    thisx = x0
    ucl[method] = np.zeros((Nsim,Nu))
    # ysp = np.tile(xs,(Nsim+1,1))

    xsp = np.zeros((Nsim+1,Nx))
    usp = np.zeros((Nsim,Nu))

    # ysp[int(Nsim/3):int(2*Nsim/3),:] = xs*np.array([.85,.75,1.15])
    for t in range(Nsim):
        # Figure out setpoints.
#        if t == 0 or not np.all(ysp[t,:] == ysp[t-1,:]):
#            thisysp = ysp[t,:]
#            sstargs[method].fixvar("y",0,thisysp[contVars],contVars)
#            sstargs[method].guess["u",0] = us
#            sstargs[method].guess["x",0] = thisysp
#            sstargs[method].guess["y",0] = thisysp
#            sstargs[method].solve()
#
#            print "%10s %3d: %s" % ("sstarg",t,sstargs[method].stats["status"])
#            if sstargs[method].stats["status"] != "Solve_Succeeded":
#                print "***Target finder failed!"
#                break
#
#            xsp[t,:] = np.squeeze(sstargs[method].var["x",0])
#            usp[t,:] = np.squeeze(sstargs[method].var["u",0])
#
#            solvers[method].par["x_sp"] = [xsp[t,:]]*(Nt + 1)
#            solvers[method].par["u_sp"] = [usp[t,:]]*Nt

        # Fix initial condition and solve.
        t0 = time.time()
        solvers[method].fixvar("x",0,thisx)
        solvers[method].solve()
        print "%10s %3d: %s" % (method,t,solvers[method].stats["status"])
        if solvers[method].stats["status"] != "Solve_Succeeded":
            print "***Solver failed!"
            break
        else:
            solvers[method].saveguess()

        thisu = np.squeeze(solvers[method].var["u"][0])
        ucl[method][t,:] = thisu
        t1 = time.time()
        tcl[method][t] = t1-t0
        thisx = cstr.sim(thisx,thisu,ds)
        xcl[method][t+1,:] = thisx

        # Update previous u.
        solvers[method].par["u_prev",0] = ucl[method][t,:]

# Define plotting function.
def cstrplot(x,u,xsp=None,contVars=[],title=None,colors={},labels={},
             markers={},keys=None,bounds=None,ilegend=0):
    if keys is None:
        keys = x.keys()
    for k in keys:
        u[k] = np.concatenate((u[k],u[k][-1:,:]))
    ylabelsx = ["$c$ (mol/L)", "$T$ (K)", "$h$ (m)"]
    ylabelsu = ["$T_c$ (K)", "$F$ (kL/min)"]

    gs = gridspec.GridSpec(Nx*Nu,2)
    fig = plt.figure(figsize=(10,6),facecolor="none")
    leglines = []
    leglabels = []
    for i in range(Nx):
        ax = fig.add_subplot(gs[i*Nu:(i+1)*Nu,0])
        for k in keys:
            t = np.arange(0,x[k].shape[0])*Delta
            args = {"color":colors.get(k,"black"), "label":labels.get(k,k),
                    "marker":markers.get(k,"")}
            [line] = ax.plot(t,x[k][:,i],markeredgecolor="none",**args)
            if i == ilegend:
                leglines.append(line)
                leglabels.append(args["label"])
        if i in contVars and xsp is not None:
            ax.step(t,xsp[:,i],linestyle="--",color="black",where="post")
        ax.set_ylabel(ylabelsx[i])
        mpc.plots.zoomaxis(ax,yscale=1.1)
        mpc.plots.prettyaxesbox(ax)
        mpc.plots.prettyaxesbox(ax,
            facecolor="white",front=False)
    ax.set_xlabel("Time (min)")
    for i in range(Nu):
        ax = fig.add_subplot(gs[i*Nx:(i+1)*Nx,1])
        for k in keys:
            t = np.arange(0,u[k].shape[0])*Delta
            args = {"color":colors.get(k,"black"), "label":labels.get(k,k)}
            ax.step(t,u[k][:,i],where="post",**args)
        if bounds is not None:
            for b in set(["uub", "ulb"]).intersection(bounds.keys()):
                ax.plot(np.array([t[0],t[-1]]),np.ones((2,))*bounds[b][i],
                        '--k')
        ax.set_ylabel(ylabelsu[i])
        mpc.plots.zoomaxis(ax,yscale=1.25)
        mpc.plots.prettyaxesbox(ax)
        mpc.plots.prettyaxesbox(ax,
            facecolor="white",front=False)
    ax.set_xlabel("Time (min)")
    fig.legend(leglines,leglabels,loc="lower center",ncol=len(keys))
    fig.tight_layout(pad=.5,rect=(0,.075,1,1))
    if title is not None:
        fig.canvas.set_window_title(title)
    return fig

x = xcl['nmpc']
u = ucl['nmpc']
ptimes = tcl['nmpc']
# Make plots.
keys = ["uncont", "nmpc"]
colors = {"lmpc":"blue", "nmpc":"green", "uncont":"red"}
labels = {"lmpc":"LMPC", "nmpc":"NMPC", "uncont":"Uncontrolled"}
markers = {"lmpc":"s", "nmpc":"o", "uncont":"^"}
plotbounds = dict([(k,bounds[k][0]) for k in ["ulb","uub"]])
fig = cstrplot(xcl, ucl, colors=colors, contVars=contVars, labels=labels,
               keys=keys, markers={}, bounds=plotbounds, ilegend=2)
fig.show()
# mpc.plots.showandsave(fig,"cstr_startup.pdf",facecolor="none")


"""
===============================================
Cross-validation on diabetes Dataset Exercise
===============================================

A tutorial exercise which uses cross-validation with linear models.

This exercise is used in the :ref:`cv_estimators_tut` part of the
:ref:`model_selection_tut` section of the :ref:`stat_learn_tut_index`.
"""
from __future__ import print_function
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn import cross_validation, datasets, linear_model

diabetes = datasets.load_diabetes()
X = diabetes.data[:150]
y = diabetes.target[:150]

lasso = linear_model.Lasso()
alphas = np.logspace(-4, -.5, 30)

scores = list()
scores_std = list()

for alpha in alphas:
    lasso.alpha = alpha
    this_scores = cross_validation.cross_val_score(lasso, X, y, n_jobs=1)
    scores.append(np.mean(this_scores))
    scores_std.append(np.std(this_scores))

plt.figure(figsize=(4, 3))
plt.semilogx(alphas, scores)
# plot error lines showing +/- std. errors of the scores
plt.semilogx(alphas, np.array(scores) + np.array(scores_std) / np.sqrt(len(X)),
             'b--')
plt.semilogx(alphas, np.array(scores) - np.array(scores_std) / np.sqrt(len(X)),
             'b--')
plt.ylabel('CV score')
plt.xlabel('alpha')
plt.axhline(np.max(scores), linestyle='--', color='.5')

##############################################################################
# Bonus: how much can you trust the selection of alpha?

# To answer this question we use the LassoCV object that sets its alpha
# parameter automatically from the data by internal cross-validation (i.e. it
# performs cross-validation on the training data it receives).
# We use external cross-validation to see how much the automatically obtained
# alphas differ across different cross-validation folds.
lasso_cv = linear_model.LassoCV(alphas=alphas)
k_fold = cross_validation.KFold(len(X), 3)

print("Answer to the bonus question:",
      "how much can you trust the selection of alpha?")
print()
print("Alpha parameters maximising the generalization score on different")
print("subsets of the data:")
for k, (train, test) in enumerate(k_fold):
    lasso_cv.fit(X[train], y[train])
    print("[fold {0}] alpha: {1:.5f}, score: {2:.5f}".
          format(k, lasso_cv.alpha_, lasso_cv.score(X[test], y[test])))
print()
print("Answer: Not very much since we obtained different alphas for different")
print("subsets of the data and moreover, the scores for these alphas differ")
print("quite substantially.")

plt.show()


"""This module is for testing stocks"""
from unittest import mock
from django.test import TestCase
from stocks.models import Stock, DailyStockQuote
import pandas as pd
from yahoo_historical import Fetcher
from authentication.plaid_middleware import PlaidMiddleware
import pytest

class StocksViewTests(TestCase):
    """
    Testing Stocks Model
    """
    @classmethod
    def setup_class(cls):
        """Setting up testing"""
        cls._original_init_method = Fetcher.__init__
        Fetcher.__init__ = mock.Mock(return_value=None)
        PlaidMiddleware.__call__ = lambda self, request: self.get_response(request)

    @classmethod
    def teardown_class(cls):
        """Teardown testing"""
        Fetcher.__init__ = cls._original_init_method

    @mock.patch.object(
        Fetcher,
        'getHistorical',
        mock.MagicMock(return_value=pd.DataFrame({
            'Close': [1.5, 2.5],
            'Date': ["2017-05-05", "2017-05-06"],
        }))
    )
    @pytest.mark.django_db(transaction=True)
    def test_api_for_real_stock(self):
        """
        Testing adding stock via endpoint, asserting stock is inserted
        """
        ticker = "googl"
        name = "Google"
        data = {'name': name, 'ticker': ticker}
        request = self.client.post('/stocks/addstock/', data)
        self.assertEqual(request.status_code, 200)
        data = Stock.objects.all()
        self.assertEqual(len(data), 1)

    @mock.patch.object(
        Fetcher,
        'getHistorical',
        mock.MagicMock(side_effect=KeyError('abc'))
    )
    def test_api_for_invalid_ticker(self):
        """
        Testing adding stock via endpoint, asserting stock is inserted but no
        data added to DailyStockQuote since ticker is invalid
        """
        ticker = "xxx"
        name = "Julian"
        data = {'name': name, 'ticker': ticker}
        request = self.client.post('/stocks/addstock/', data)
        self.assertEqual(request.status_code, 500)
        data = DailyStockQuote.objects.all()
        self.assertEqual(len(data), 0)

    def test_api_with_invalid_call(self):
        """
        Endpoint only works with POST
        """
        request = self.client.get('/stocks/addstock/')
        self.assertEqual(request.status_code, 405)

    @mock.patch.object(
        Fetcher,
        'getHistorical',
        mock.MagicMock(return_value=pd.DataFrame({
            'Close': [1.5, 2.5],
            'Date': ["2017-05-05", "2017-05-06"],
        }))
    )
    @pytest.mark.django_db(transaction=True)
    def test_fill_quote_history(self):
        """
        Filling data for Stock
        """
        ticker = "ibm"
        name = "IBM"
        data = {'name': name, 'ticker': ticker}
        request = self.client.get('/stocks/addstock/', data)
        stock_id = request.content
        data = DailyStockQuote.objects.filter(stock_id=stock_id)
        stock_data = Stock.objects.filter(id=stock_id)
        self.assertGreater(len(data), 0)
        self.assertEqual(len(stock_data), 1)


"""
=========================================
Image denoising using dictionary learning
=========================================

An example comparing the effect of reconstructing noisy fragments
of a raccoon face image using firstly online :ref:`DictionaryLearning` and
various transform methods.

The dictionary is fitted on the distorted left half of the image, and
subsequently used to reconstruct the right half. Note that even better
performance could be achieved by fitting to an undistorted (i.e.
noiseless) image, but here we start from the assumption that it is not
available.

A common practice for evaluating the results of image denoising is by looking
at the difference between the reconstruction and the original image. If the
reconstruction is perfect this will look like Gaussian noise.

It can be seen from the plots that the results of :ref:`omp` with two
non-zero coefficients is a bit less biased than when keeping only one
(the edges look less prominent). It is in addition closer from the ground
truth in Frobenius norm.

The result of :ref:`least_angle_regression` is much more strongly biased: the
difference is reminiscent of the local intensity value of the original image.

Thresholding is clearly not useful for denoising, but it is here to show that
it can produce a suggestive output with very high speed, and thus be useful
for other tasks such as object classification, where performance is not
necessarily related to visualisation.

"""
print(__doc__)

from time import time

import matplotlib.pyplot as plt
import numpy as np
import scipy as sp

from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.feature_extraction.image import extract_patches_2d
from sklearn.feature_extraction.image import reconstruct_from_patches_2d
from sklearn.utils.testing import SkipTest
from sklearn.utils.fixes import sp_version

if sp_version < (0, 12):
    raise SkipTest("Skipping because SciPy version earlier than 0.12.0 and "
                   "thus does not include the scipy.misc.face() image.")

###############################################################################
try:
    from scipy import misc
    face = misc.face(gray=True)
except AttributeError:
    # Old versions of scipy have face in the top level package
    face = sp.face(gray=True)

# Convert from uint8 representation with values between 0 and 255 to
# a floating point representation with values between 0 and 1.
face = face / 255

# downsample for higher speed
face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2]
face /= 4.0
height, width = face.shape

# Distort the right half of the image
print('Distorting image...')
distorted = face.copy()
distorted[:, width // 2:] += 0.075 * np.random.randn(height, width // 2)

# Extract all reference patches from the left half of the image
print('Extracting reference patches...')
t0 = time()
patch_size = (7, 7)
data = extract_patches_2d(distorted[:, :width // 2], patch_size)
data = data.reshape(data.shape[0], -1)
data -= np.mean(data, axis=0)
data /= np.std(data, axis=0)
print('done in %.2fs.' % (time() - t0))

###############################################################################
# Learn the dictionary from reference patches

print('Learning the dictionary...')
t0 = time()
dico = MiniBatchDictionaryLearning(n_components=100, alpha=1, n_iter=500)
V = dico.fit(data).components_
dt = time() - t0
print('done in %.2fs.' % dt)

plt.figure(figsize=(4.2, 4))
for i, comp in enumerate(V[:100]):
    plt.subplot(10, 10, i + 1)
    plt.imshow(comp.reshape(patch_size), cmap=plt.cm.gray_r,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
plt.suptitle('Dictionary learned from face patches\n' +
             'Train time %.1fs on %d patches' % (dt, len(data)),
             fontsize=16)
plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23)

###############################################################################
# Display the distorted image

def show_with_diff(image, reference, title):
    """Helper function to display denoising"""
    plt.figure(figsize=(5, 3.3))
    plt.subplot(1, 2, 1)
    plt.title('Image')
    plt.imshow(image, vmin=0, vmax=1, cmap=plt.cm.gray,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
    plt.subplot(1, 2, 2)
    difference = image - reference

    plt.title('Difference (norm: %.2f)' % np.sqrt(np.sum(difference ** 2)))
    plt.imshow(difference, vmin=-0.5, vmax=0.5, cmap=plt.cm.PuOr,
               interpolation='nearest')
    plt.xticks(())
    plt.yticks(())
    plt.suptitle(title, size=16)
    plt.subplots_adjust(0.02, 0.02, 0.98, 0.79, 0.02, 0.2)

show_with_diff(distorted, face, 'Distorted image')

###############################################################################
# Extract noisy patches and reconstruct them using the dictionary

print('Extracting noisy patches... ')
t0 = time()
data = extract_patches_2d(distorted[:, width // 2:], patch_size)
data = data.reshape(data.shape[0], -1)
intercept = np.mean(data, axis=0)
data -= intercept
print('done in %.2fs.' % (time() - t0))

transform_algorithms = [
    ('Orthogonal Matching Pursuit\n1 atom', 'omp',
     {'transform_n_nonzero_coefs': 1}),
    ('Orthogonal Matching Pursuit\n2 atoms', 'omp',
     {'transform_n_nonzero_coefs': 2}),
    ('Least-angle regression\n5 atoms', 'lars',
     {'transform_n_nonzero_coefs': 5}),
    ('Thresholding\n alpha=0.1', 'threshold', {'transform_alpha': .1})]

reconstructions = {}
for title, transform_algorithm, kwargs in transform_algorithms:
    print(title + '...')
    reconstructions[title] = face.copy()
    t0 = time()
    dico.set_params(transform_algorithm=transform_algorithm, **kwargs)
    code = dico.transform(data)
    patches = np.dot(code, V)

    patches += intercept
    patches = patches.reshape(len(data), *patch_size)
    if transform_algorithm == 'threshold':
        patches -= patches.min()
        patches /= patches.max()
    reconstructions[title][:, width // 2:] = reconstruct_from_patches_2d(
        patches, (height, width // 2))
    dt = time() - t0
    print('done in %.2fs.' % dt)
    show_with_diff(reconstructions[title], face,
                   title + ' (time: %.1fs)' % dt)

plt.show()


import numpy as np
import pandas

def normalize_features(array):
   """
   Normalize the features in our data set.
   """
   array_normalized = (array-array.mean())/array.std()
   mu = array.mean()
   sigma = array.std()

   return array_normalized, mu, sigma

def compute_cost(features, values, theta):
    """
    Compute the cost function given a set of features / values, and the values for our thetas.

    This should be the same code as the compute_cost function in the lesson #3 exercises. But
    feel free to implement your own.
    """

    # your code here
    m = len(values)
    sum_of_square_errors = np.square(np.dot(features, theta) - values).sum()
    cost = sum_of_square_errors / (2*m)

    return cost

def gradient_descent(features, values, theta, alpha, num_iterations):
    """
    Perform gradient descent given a data set with an arbitrary number of features.

    This is the same gradient descent code as in the lesson #3 exercises. But feel free
    to implement your own.
    """
    m = len(values)
    cost_history = []

    for i in range(num_iterations):
        # your code here
        # Calculate cost
        cost = compute_cost(features, values, theta)

        # Append cost to history
        cost_history.append(cost)

        # Calculate new theta
        theta = theta + (alpha / m) * np.dot((values - np.dot(features,theta)),features)
    return theta, pandas.Series(cost_history)

def predictions(dataframe):
    '''
    The NYC turnstile data is stored in a pandas dataframe called weather_turnstile.
    Using the information stored in the dataframe, lets predict the ridership of
    the NYC subway using linear regression with gradient descent.

    You can look at information contained in the turnstile weather dataframe
    at the link below:
    https://www.dropbox.com/s/meyki2wl9xfa7yk/turnstile_data_master_with_weather.csv

    Your prediction should have a R^2 value of .40 or better.

    Note: due to the memory and CPU limitation of our amazon EC2 instance, we will
    give you a random subet (~15%) of the data contained in turnstile_data_master_with_weather.csv

    If you receive a "server has encountered an error" message, that means you are hitting
    the 30 second  limit that's placed on running your program. Try using a smaller number
    for num_iterations if that's the case.

    Or if you are using your own algorithm/modesl, see if you can optimize your code so it
    runs faster.
    '''

    dummy_units = pandas.get_dummies(dataframe['UNIT'], prefix='unit')
    features = dataframe[['rain', 'precipi', 'Hour', 'meantempi']].join(dummy_units)
    values = dataframe[['ENTRIESn_hourly']]
    m = len(values)

    features, mu, sigma = normalize_features(features)

    features['ones'] = np.ones(m)
    features_array = np.array(features)
    values_array = np.array(values).flatten()

    #Set values for alpha, number of iterations.
    alpha = 0.1 # please feel free to play with this value
    num_iterations = 75 # please feel free to play with this value

    #Initialize theta, perform gradient descent
    theta_gradient_descent = np.zeros(len(features.columns))
    theta_gradient_descent, cost_history = gradient_descent(features_array, values_array, theta_gradient_descent,
                                                            alpha, num_iterations)
    #print cost_history
    prediction = np.dot(features_array, theta_gradient_descent)

    return prediction


import numpy as np
import get_fit_1D_open_manifold_3D_param
import solve_d_R_d_tau_p_3D
import a
from scipy.io import loadmat
import matplotlib.pyplot as plt

#import matplotlib.pyplot as plt

'''
function [a,b,tau] = fit_1D_open_manifold_3D(psi)
%
% fit_1D_open_manifold_3D
%
% fit the eigenvectors for a 1D open manifold to the model
% x_ij = a_j cos(j*pi*tau_i) + b_j.
%
% j goes from 1 to 3 (this is only for 3D systems).
%
% i goes from 1 to nS where nS is the number of data points to be fitted.
%
% For a fixed set of a_j and b_j, j=1:3, tau_i for i=1:nS are
% obtained by putting dR/d(tau_i) to zero.
%
% For a fixed set of tau_i, i=1:nS, a_j and b_j for j=1:3 are
% obtained by solving 3 sets of 2x2 linear equations.
%
% Fit parameters and initial set of {\tau} are specified in
%
%   get_fit_1D_open_manifold_3D_param.m
%
% copyright (c) Russell Fung 2014
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  global p nDim a b x x_fit

  '''
'''
def plot_fitted_curve(hFig):
    global x x_fit
    h = plt.figure(hFig)
    hsp = plt.subplot(2,2,1)
    plot3(x(:,1),x(:,2),x(:,3),'b.','lineWidth',1);
  hold on
  plot3(x_fit(:,1),x_fit(:,2),x_fit(:,3),'g.','lineWidth',1);
  hold off
  set(hsp,'lineWidth',2,'fontSize',15);
  hsp = subplot(2,2,2);
  plotRF(hsp,x(:,1),x(:,2),'','','','b.');
  addplotRF(hsp,x_fit(:,1),x_fit(:,2),'g.');
  hsp = subplot(2,2,3);
  plotRF(hsp,x(:,1),x(:,3),'','','','b.');
  addplotRF(hsp,x_fit(:,1),x_fit(:,3),'g.');
  hsp = subplot(2,2,4);
  plotRF(hsp,x(:,2),x(:,3),'','','','b.');
  addplotRF(hsp,x_fit(:,2),x_fit(:,3),'g.');
  drawnow
%end
'''
eps = 1e-4

#global maxIter,delta_a_max, delta_b_max,delta_tau_max,a_b_tau_result

def op(psi):
    a.init()
    #global p, nDim, a, b, x, x_fit
    a.nDim = 3
    #tau = get_fit_1D_open_manifold_3D_param
    tau = get_fit_1D_open_manifold_3D_param.op(psi)
    aux = np.zeros((tau.shape[0],5))   #added
    nS = a.x.shape[0]

    for iter in xrange(1,a.maxIter+1):
        string ='iteration ' + str(iter)
        print string
        '''
        #%%%%%%%%%%%%%%%%%%%%%
        #% solve for a and b %
        #%%%%%%%%%%%%%%%%%%%%%
        '''
        a_old = a.a
        b_old = a.b
        j_pi_tau = np.dot(tau,np.pi*np.array([[1,2,3]]))
        cos_j_pi_tau = np.cos(j_pi_tau)
        A11 = np.sum(cos_j_pi_tau**2, axis=0)
        A12 = np.sum(cos_j_pi_tau, axis=0)
        A21 = A12
        A22 = nS
        x_cos_j_pi_tau = a.x*cos_j_pi_tau
        b1 = np.sum(x_cos_j_pi_tau, axis=0)
        b2 = np.sum(a.x, axis=0)
        coeff = np.zeros((2,3))
        for qq in xrange(3):
            A = np.array([[A11[qq],A12[qq]],[A21[qq], A22]])
            b = np.array([b1[qq], b2[qq]])
            coeff[:,qq] = np.linalg.solve(A,b)

        a.a = coeff[0,:]
        a.b = coeff[1,:]
        '''
        %%%%%%%%%%%%%%%%%%%%%%%%%
        #% plot the fitted curve %
        %%%%%%%%%%%%%%%%%%%%%%%%%
        '''
        j_pi_tau = np.dot(np.linspace(0,1,1000).reshape(-1,1),np.array([[1,2,3]]))*np.pi
        cos_j_pi_tau = np.cos(j_pi_tau)
        tmp = a.a*cos_j_pi_tau
        a.x_fit = tmp + a.b
        #%plot_fitted_curve(iter)
        '''
        %%%%%%%%%%%%%%%%%
        #% solve for tau %
        %%%%%%%%%%%%%%%%%
        '''
        tau_old = tau
        for a.p in xrange(nS):
          tau[a.p],beta = solve_d_R_d_tau_p_3D.op()   #added
          for kk in xrange(beta.shape[0]):
              aux[a.p,kk] = beta[kk]
        '''
        if iter == 0:
            data = loadmat('aux0.mat')  # (this is for < v7.3
        elif iter == 1:
            data = loadmat('aux1.mat')  # (this is for < v7.3
        else:
            data = loadmat('aux2.mat')  # (this is for < v7.3
        imaux = data['aux']
        plt.subplot(2, 2, 1)
        plt.imshow(aux, cmap=plt.get_cmap('gray'),aspect=0.1)
        plt.title('aux')
        plt.subplot(2, 2, 2)
        plt.imshow(imaux, cmap=plt.get_cmap('gray'), aspect=0.1)
        plt.title('imaux')
        plt.show()
        '''

        '''
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        #% calculate the changes in fitting parameters %
        #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        '''
        delta_a = np.fabs(a.a-a_old)/(np.fabs(a.a)+eps)
        delta_b = np.fabs(a.b-b_old)/(np.fabs(a.b)+eps)
        delta_tau = np.fabs(tau-tau_old)
        delta_a = max(delta_a)*100
        delta_b = max(delta_b)*100
        delta_tau = max(delta_tau)
        print '  changes in fitting parameters: \n'
        string = '  amplitudes: '+ str(delta_a) + '\n' + \
                 '  offsets: ' + str(delta_b) + ' \n' +\
                 '  values of tau: ' + str(delta_tau) + ' \n'
        print string
        if (delta_a<a.delta_a_max) and (delta_b < a.delta_b_max) and (delta_tau < a.delta_tau_max):
            break

    return (a.a,a.b,tau)


# -*- coding: utf-8 -*-

import os
import re
import matplotlib.pyplot as plt

from framework.mongo import database
from website import settings
from website.app import init_app

from .utils import plot_dates, oid_to_datetime, mkdirp

log_collection = database['nodelog']

FIG_PATH = os.path.join(settings.ANALYTICS_PATH, 'figs', 'addons')
mkdirp(FIG_PATH)

ADDONS = [
    'box',
    'dataverse',
    'dropbox',
    'figshare',
    'github',
    'googledrive',
    'mendeley',
    's3',
    'zotero',
]

def get_collection_datetimes(collection, _id='_id', query=None):
    query = query or {}
    return [
        oid_to_datetime(record[_id])
        for record in collection.find({}, {_id: True})
    ]

def analyze_model(model):

    dates = get_collection_datetimes(model._storage[0].store)
    return {
        'dates': dates,
        'count': len(dates),
    }

def analyze_addon_installs(name):

    config = settings.ADDONS_AVAILABLE_DICT[name]

    results = {
        key: analyze_model(model)
        for key, model in config.settings_models.iteritems()
    }

    return results

def analyze_addon_logs(name):

    pattern = re.compile('^{0}'.format(name), re.I)
    logs = log_collection.find({'action': {'$regex': pattern}}, {'date': True})
    return [
        record['date']
        for record in logs
    ]

def analyze_addon(name):

    installs = analyze_addon_installs(name)
    for model, result in installs.iteritems():
        if not result['dates']:
            continue
        fig = plot_dates(result['dates'])
        plt.title('{} configurations: {} ({} total)'.format(name, model, len(result['dates'])))
        plt.savefig(os.path.join(FIG_PATH, '{}-installs-{}.png'.format(name, model)))
        plt.close()

    log_dates = analyze_addon_logs(name)
    if not log_dates:
        return
    fig = plot_dates(log_dates)
    plt.title('{} actions ({} total)'.format(name, len(log_dates)))
    plt.savefig(os.path.join(FIG_PATH, '{}-actions.png'.format(name)))
    plt.close()

def main():
    init_app(routes=False)
    for addon in ADDONS:
        if addon in settings.ADDONS_AVAILABLE_DICT:
            analyze_addon(addon)

if __name__ == '__main__':
    main()


#  Copyright 2016 Intel Corporation
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from brainiak.fcma.mvpa_voxelselector import MVPAVoxelSelector
from brainiak.fcma.preprocessing import prepare_searchlight_mvpa_data
from brainiak import io
from sklearn import svm
import sys
from mpi4py import MPI
import logging
import nibabel as nib
import numpy as np
from brainiak.searchlight.searchlight import Searchlight

format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
# if want to output log to a file instead of outputting log to the console,
# replace "stream=sys.stdout" with "filename='fcma.log'"
logging.basicConfig(level=logging.INFO, format=format, stream=sys.stdout)
logger = logging.getLogger(__name__)

"""
example running command in run_mvpa_voxel_selection.sh
"""
if __name__ == '__main__':
    if MPI.COMM_WORLD.Get_rank()==0:
        logger.info(
            'programming starts in %d process(es)' %
            MPI.COMM_WORLD.Get_size()
        )
    if len(sys.argv) != 6:
        logger.error('the number of input argument is not correct')
        sys.exit(1)

    data_dir = sys.argv[1]
    suffix = sys.argv[2]
    mask_file = sys.argv[3]
    epoch_file = sys.argv[4]

    # all MPI processes read the mask; the mask file is small
    mask_image = nib.load(mask_file)
    mask = io.load_boolean_mask(mask_file)
    data = None
    labels = None
    if MPI.COMM_WORLD.Get_rank()==0:
        logger.info(
            'mask size: %d' %
            np.sum(mask)
        )
        images = io.load_images_from_dir(data_dir, suffix=suffix)
        conditions = io.load_labels(epoch_file)
        data, labels = prepare_searchlight_mvpa_data(images, conditions)

        # setting the random argument produces random voxel selection results
        # for non-parametric statistical analysis.
        # There are three random options:
        # RandomType.NORANDOM is the default
        # RandomType.REPRODUCIBLE permutes the voxels in the same way every run
        # RandomType.UNREPRODUCIBLE permutes the voxels differently across runs
        # example
        #from brainiak.fcma.preprocessing import RandomType
        #data, labels = prepare_searchlight_mvpa_data(images, conditions,
        #                                                    random=RandomType.UNREPRODUCIBLE)

        # the following line is an example to leaving a subject out
        #epoch_info = [x for x in epoch_info if x[1] != 0]

    num_subjs = int(sys.argv[5])
    # create a Searchlight object
    sl = Searchlight(sl_rad=1)
    mvs = MVPAVoxelSelector(data, mask, labels, num_subjs, sl)
    clf = svm.SVC(kernel='linear', shrinking=False, C=1, gamma='auto')
    # only rank 0 has meaningful return values
    score_volume, results = mvs.run(clf)
    # this output is just for result checking
    if MPI.COMM_WORLD.Get_rank()==0:
        score_volume = np.nan_to_num(score_volume.astype(np.float))
        io.save_as_nifti_file(score_volume, mask_image.affine,
                                   'result_score.nii.gz')
        seq_volume = np.zeros(mask.shape, dtype=np.int)
        seq = np.zeros(len(results), dtype=np.int)
        with open('result_list.txt', 'w') as fp:
            for idx, tuple in enumerate(results):
                fp.write(str(tuple[0]) + ' ' + str(tuple[1]) + '\n')
                seq[tuple[0]] = idx
        seq_volume[mask] = seq
        io.save_as_nifti_file(seq_volume, mask_image.affine,
                                   'result_seq.nii.gz')


# Import any required libraries or modules.
import numpy as np
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
import csv
import sys

class MetaParams:
    n_lda_ensemble = 101
    lda_ensemble_feature_fraction = 0.4
    mode = 'lda_ensemble'

# The following is a hacky container for Statistics computed from the
# whole training set; we don't want to have to recompute them again at every call
# to build_template (it becomes slow for parameter searches with cross validation),
# so we preserve it here between calls. The proper place to
# do this would be in main.py, but we don't want to touch that.
Global = lambda: None
Global.ready = False

def pca_converter(data, feature_discriminabilities, explained_variance):
    '''
    PCA conversion of the data. The PCA is based on the complete dataset, but each feature
        is normalized to a std dev proportional to the given discriminability.
    :param data: n_samples x n_features matrix with all data to do PCA on
    :param feature_discriminabilities: n_features length vector
    :param explained_variance: ratio of explained variance (between 0 and 1) that will
        determine how many components are kept
    :return: function transforming data into pca components, and covariance matrix
        of transformed data
    '''
    mu = np.mean(data, axis=0)
    std = np.std(data, axis=0) / feature_discriminabilities
    normalized_data = (data - mu) / std
    u, s, vt = np.linalg.svd(normalized_data)
    cut_idx = np.argmin(np.abs(np.cumsum(s * s) / np.sum(s * s) - explained_variance))
    vt = vt[:cut_idx + 1]
    return (lambda x, mu=mu, std=std, vt=vt: np.dot((x - mu) / std, vt.T)),\
           np.diag(s[:cut_idx + 1] ** 2 / (len(data) - 1))

def preprocess_data(data):
    '''
    Turn raw data into an array of hand-picked features useful for classification
    :param data: n_samples x n_raw_features numpy array
    :return: n_samples x n_processed_features array
    '''
    keypress_dt = data[:, 8::10] - data[:, 3::10]  # duration of each keystroke
    key_to_key_dt = data[:, 13::10] - data[:, 3:-10:10]  # interval between keystrokes
    x_down = data[:, 4::10].astype(np.float) / data[:, 1][:, None].astype(np.float)  # x relative to screen width
    y_down = data[:, 5::10].astype(np.float) / data[:, 0][:, None].astype(np.float)  # y relative to screen height
    x_up = data[:, 9::10].astype(np.float) / data[:, 1][:, None].astype(np.float)  # x relative to screen width
    y_up = data[:, 10::10].astype(np.float) / data[:, 0][:, None].astype(np.float)  # y relative to screen height
    size_down = data[:, 6::10]
    size_up = data[:, 11::10]
    pressure_down = data[:, 7::10]
    pressure_up = data[:, 12::10]

    assert np.all((x_down >= 0) & (x_down <= 1) & (y_down >= 0) & (y_down <= 1))
    assert np.all((x_up >= 0) & (x_up <= 1) & (y_up >= 0) & (y_up <= 1))
    touch_d = np.hypot(x_down - x_up, y_down - y_up)
    collected_data = np.hstack((keypress_dt, key_to_key_dt,
                                np.diff(x_down, axis=1), np.diff(y_down, axis=1),
                                touch_d,
                                size_down, size_up, pressure_down, pressure_up,
                                ))
    return collected_data

def get_random_feature_selector(n_all_features, feature_fraction, seed):
    '''
    Return a selector of random features from a data array
    :param n_all_features: total number of features
    :param feature_fraction: desired fraction of selected features
    :param seed: random seed for repeatable experiments
    :return: a function taking in full data and returning only the random features from it
    '''
    n_features = int(np.round(feature_fraction * n_all_features))
    rng = np.random.RandomState(seed)
    p = rng.permutation(n_all_features)[:n_features]
    return lambda x, p=p: x[..., p]

def simple_gaussian(user_pca):
    # template will consist of mean and std dev of each feature in pca space
    mean_pca = np.mean(user_pca, axis=0)
    std_pca = np.std(user_pca, axis=0)
    return mean_pca, std_pca

def scikit_classifier(user, training_dataset, generator=lambda:KNeighborsClassifier(5)):
    '''
    Train a given classifier on user vs others
    :param generator: a function creating a scikit classifier with fit and predict functions
    :return: the trained classifier
    '''
    all_users = training_dataset.keys()
    others_raw = np.vstack([training_dataset[u] for u in all_users if u != user])
    others_pca = Global.pca(preprocess_data(others_raw))
    user_raw = training_dataset[user]
    user_pca = Global.pca(preprocess_data(user_raw))

    clf = generator()
    clf.fit(np.vstack((user_pca, others_pca)),
            np.hstack((np.zeros(len(user_pca)), np.ones(len(others_pca)))))
    return clf

def lda(user_pca, all_pca_cov, n_all):
    '''
    Compute the Fisher discriminant vector and threshold to classify user vs others.
    :param user_pca: n_samples x n_pca_features array of user instances
    :param all_pca_cov: covariance matrix of the complete dataset; it is assumed that
        the user data was part of the dataset, and that the mean of the whole dataset
        is 0 for every feature
    :param n_all: number of samples that formed the complete dataset
    :return: Fisher discriminant vector, threshold
    '''
    n_user = len(user_pca)
    assert n_user < n_all - 1  # make sure the complete dataset has more than just the current user

    # We compute mean and variance for the user data directly, and infer the mean
    # and variance of the rest of the dataset from the covariance of the complete set
    # (and its mean, which is assumed zero)
    user_mu = np.mean(user_pca, axis=0)
    others_mu = - n_user * user_mu / (n_all - n_user)
    user_sigma = np.cov(user_pca.T)
    def sq_(x):
        return x[:, None] * x[None, :]
    others_sigma = ((n_all - 1) * all_pca_cov - (n_user - 1) * user_sigma\
                   - n_user * sq_(user_mu) - (n_all - n_user) * sq_(others_mu)) / (n_all - n_user - 1)

    ld_vector = np.dot(np.linalg.inv(user_sigma + others_sigma), user_mu - others_mu)  # order determines sign of criterion
    ld_vector /= np.linalg.norm(ld_vector)

    # find the threshold for equal false positives and false negatives
    user_proj_mu = np.dot(user_mu, ld_vector)
    others_proj_mu = np.dot(others_mu, ld_vector)
    user_proj_std = np.sqrt(np.dot(ld_vector, np.dot(user_sigma, ld_vector)))
    others_proj_std = np.sqrt(np.dot(ld_vector, np.dot(others_sigma, ld_vector)))
    ld_threshold = (others_proj_std * user_proj_mu + user_proj_std * others_proj_mu) / (user_proj_std + others_proj_std)
    return ld_vector, ld_threshold

def compute_feature_discriminabilities(each_preprocessed):
    '''
    Return a vector of discriminability for each feature
    :param each_preprocessed: list with one n_samples x n_features data matrix for each user
    :return: vector of discriminabilities (sqrt of the square of the difference of means divided by
        the sum of variances) for each feature
    '''
    n_users = len(each_preprocessed)
    each_mu = np.array([np.mean(m, axis=0) for m in each_preprocessed])  # n_users x n_features
    each_var = np.array([np.var(m, axis=0) for m in each_preprocessed])  # n_users x n_features
    # compute discriminability for each feature and pair of users
    pairwise_discriminability = (each_mu[:, None, :] - each_mu[None :, :]) ** 2 / (1e-6 + each_var[:, None, :] + each_var[None :, :])
    # compute discriminability of each feature as the average over pairs of users
    return np.sqrt(np.sum(pairwise_discriminability, axis=(0, 1)) / (n_users * (n_users - 1)))

def _prepare_global(training_dataset):
    '''
    Processing of the complete dataset, to be reused for each user
        - feature preprocessing
        - pca converter
        - selection of features and computation of covariances for ensemble lda
    :param training_dataset: the complete dataset
    :return: None. The Global container is initialized with all necessary data
    '''
    each_preprocessed = [preprocess_data(training_dataset[u]) for u in training_dataset]
    Global.feature_discriminabilities = compute_feature_discriminabilities(each_preprocessed)

    all_preprocessed = np.vstack(each_preprocessed)
    Global.n_all = len(all_preprocessed)
    Global.pca, Global.all_pca_cov = pca_converter(all_preprocessed, Global.feature_discriminabilities, explained_variance=0.98)
    if MetaParams.mode == 'lda_ensemble':
        Global.lda_ensemble = []
        for i in range(MetaParams.n_lda_ensemble):
            seed = np.random.randint(200000)
            feature_selector = get_random_feature_selector(all_preprocessed.shape[1],
                                                           feature_fraction=MetaParams.lda_ensemble_feature_fraction, seed=seed)
            selected_pca, selected_pca_cov = pca_converter(feature_selector(all_preprocessed),
                                                           feature_selector(Global.feature_discriminabilities),
                                                           explained_variance=0.99)
            Global.lda_ensemble.append({'selector': feature_selector, 'pca': selected_pca, 'pca_cov': selected_pca_cov})
    Global.ready = True

# Implement template building here.  Feel free to write any helper classes or functions required.
# Return the generated template for that user.
def build_template(user, training_dataset):
    if not Global.ready:
        _prepare_global(training_dataset)

    user_raw = training_dataset[user]
    user_preprocessed = preprocess_data(user_raw)

    template = {}
    if MetaParams.mode in ['lda', 'simple', 'combined']:
        user_pca = Global.pca(user_preprocessed)
        template['mean_pca'], template['std_pca'] = simple_gaussian(user_pca)
        template['ld_vector'], template['ld_threshold'] =\
            lda(user_pca, all_pca_cov=Global.all_pca_cov, n_all=Global.n_all)

    if MetaParams.mode == 'lda_ensemble':
        lda_ensemble = []
        for lda_item in Global.lda_ensemble:
            user_selected_pca = lda_item['pca'](lda_item['selector'](user_preprocessed))
            ld_vector, ld_threshold = lda(user_selected_pca, n_all=Global.n_all, all_pca_cov=lda_item['pca_cov'])
            lda_ensemble.append({'ld_vector': ld_vector, 'ld_threshold': ld_threshold})
        template['lda_ensemble'] = lda_ensemble

    if MetaParams.mode in ['nonlinear', 'combined']:
        template['clf_1'] = scikit_classifier(user, training_dataset, generator=lambda: KNeighborsClassifier(5))
        template['clf_2'] = scikit_classifier(user, training_dataset, generator=lambda: svm.LinearSVC(C=0.05, class_weight='balanced'))
    return template

# Implement authentication method here.  Feel free to write any helper classes or functions required.
# Return the authenttication score and threshold above which you consider it being a correct user.
def authenticate(instance, user, templates):
    mode = MetaParams.mode
    assert mode in ['lda', 'combined', 'lda_ensemble', 'nonlinear', 'simple'], ("Unrecognized mode: %s" % mode)
    t = templates[user]
    batch_mode = instance.ndim > 1
    if not batch_mode:
        instance = instance[None, :]
    preprocessed_instance = preprocess_data(instance)

    if mode in ['lda', 'combined']:
        user_pca = Global.pca(preprocessed_instance)
        user_lda_proj = np.dot(user_pca, t['ld_vector'])
        lda_score, lda_thr = user_lda_proj - t['ld_threshold'], np.zeros(len(user_lda_proj))

    if mode in ['nonlinear', 'combined']:
        user_pca = Global.pca(preprocessed_instance)
        clf_score_1, clf_thr_1 = (t['clf_1'].predict(user_pca) == 0).astype(np.float), 0.5 * np.ones(len(user_pca))
        clf_score_2, clf_thr_2 = (t['clf_2'].predict(user_pca) == 0).astype(np.float), 0.5 * np.ones(len(user_pca))

    if mode == 'simple':
        user_pca = Global.pca(preprocessed_instance)
        z = (user_pca - t['mean_pca']) / t['std_pca']
        distance = np.mean(np.abs(z) ** 2, axis=1) ** 0.5
        score, thr = distance, 1.2 * np.ones(len(distance))

    if mode == 'lda_ensemble':
        ensemble_scores = np.empty((len(preprocessed_instance), len(t['lda_ensemble'])))
        for i, sub_t in enumerate(t['lda_ensemble']):
            g_item = Global.lda_ensemble[i]
            user_selected_pca = g_item['pca'](g_item['selector'](preprocessed_instance))
            user_thinned_lda_proj = np.dot(user_selected_pca, sub_t['ld_vector'])
            ensemble_scores[:, i] = user_thinned_lda_proj - sub_t['ld_threshold']

        score = np.mean(ensemble_scores > 0, axis=1)
        thr = 0.5 * np.ones(len(score))

    if mode == 'lda':
        score, thr = lda_score, lda_thr
    elif mode == 'nonlinear':
        score, thr = clf_score_1, clf_thr_1
    elif mode == 'combined':
        score = np.mean(np.vstack((lda_score > lda_thr, clf_score_1 > clf_thr_1, clf_score_2 > clf_thr_2)), axis=0)
        thr = 0.5 * np.ones(len(score))

    if not batch_mode:
        assert score.shape == (1, )
        assert thr.shape == (1, )
        score, thr = score[0], thr[0]

    return score, thr

def cross_validate(full_dataset, print_results=False):
    '''
    n-fold cross-validation of given dataset
    :param full_dataset: dictionary of raw data for each user
    :param print_results: if True, print progress messages and results
    :return: (percentage of false rejects, percentage of false accepts)
    '''
    n_folds = 5  # for cross-validation
    all_false_accept = 0
    all_false_reject = 0
    all_true_accept = 0
    all_true_reject = 0
    for i in range(n_folds):
        # split full dataset into training and validation
        training_dataset = dict()
        validation_dataset = dict()
        for u in full_dataset.keys():
            n = len(full_dataset[u])
            idx = np.round(float(n) / n_folds * np.arange(n_folds + 1)).astype(np.int)
            n_validation = np.diff(idx)
            rolled_set = np.roll(full_dataset[u], -idx[i], axis=0)
            training_dataset[u] = rolled_set[n_validation[i]:, :]
            validation_dataset[u] = rolled_set[:n_validation[i], :]

        # reset global data
        Global.ready = False
        templates = {u: build_template(u, training_dataset) for u in training_dataset}

        # For each user test authentication.
        true_accept = 0
        false_reject = 0
        true_reject = 0
        false_accept = 0
        for u in training_dataset:
            # Test false rejections.
            (score, threshold) = authenticate(validation_dataset[u], u, templates)
            true_accept += np.sum(score > threshold)
            false_reject += np.sum(score <= threshold)

            # Test false acceptance.
            for u_attacker in validation_dataset:
                if u == u_attacker:
                    continue
                (score, threshold) = authenticate(validation_dataset[u_attacker], u, templates)
                false_accept += np.sum(score > threshold)
                true_reject += np.sum(score <= threshold)

        if print_results:
            print "fold %i: false reject rate: %.1f%%, false accept rate: %.1f%%" %\
                  (i, 100. * float(false_reject) / (false_reject + true_accept),
                   100. * float(false_accept) / (false_accept + true_reject))
        all_false_accept += false_accept
        all_false_reject += false_reject
        all_true_accept += true_accept
        all_true_reject += true_reject

    false_reject_percent = 100. * float(all_false_reject) / (all_false_reject + all_true_accept)
    false_accept_percent = 100. * float(all_false_accept) / (all_false_accept + all_true_reject)

    if print_results:
        print "Total: false reject rate: %.1f%%, false accept rate: %.1f%%" % (false_reject_percent, false_accept_percent)

    return false_reject_percent, false_accept_percent

if __name__ == "__main__":
    # Reading the data into the training dataset separated by user.
    data_training_file = open('dataset_training.csv', 'rb')
    csv_training_reader = csv.reader(data_training_file, delimiter=',', quotechar='"')
    csv_training_reader.next()
    full_dataset = dict()
    for row in csv_training_reader:
        if row[0] not in full_dataset:
            full_dataset[row[0]] = np.array([]).reshape((0, len(row[1:])))
        full_dataset[row[0]] = np.vstack([full_dataset[row[0]], np.array(row[1:]).astype(float)])

    for feature_fraction in [0.4]:
        for n_lda_ensemble in [51]:
            n_trials = 10
            tot_rej = 0
            tot_acc = 0
            for _ in range(n_trials):
                MetaParams.feature_fraction = feature_fraction
                MetaParams.n_lda_ensemble = n_lda_ensemble
                rej, acc = cross_validate(full_dataset)
                tot_rej += rej
                tot_acc += acc
            print "feature fraction=%.2f, ensemble size=%i, false_rej=%.2f%%, false_acc=%.2f%%" % (feature_fraction, n_lda_ensemble, tot_rej / n_trials, tot_acc / n_trials)


#!/usr/bin/env python

###############################################################################
# Copyright 2019 The Apollo Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

import sys
from record_reader import RecordItemReader
import matplotlib.pyplot as plt
from cyber_py.record import RecordReader
from modules.canbus.proto import chassis_pb2

class SpeedDsteeringData:
    def __init__(self):
        self.last_steering_percentage = None
        self.last_speed_mps = None
        self.last_timestamp_sec = None
        self.speed_data = []
        self.d_steering_data = []

    def add(self, chassis):
        steering_percentage = chassis.steering_percentage
        speed_mps = chassis.speed_mps
        timestamp_sec = chassis.header.timestamp_sec

        if self.last_timestamp_sec is None:
            self.last_steering_percentage = steering_percentage
            self.last_speed_mps = speed_mps
            self.last_timestamp_sec = timestamp_sec
            return

        if (timestamp_sec - self.last_timestamp_sec) > 0.02:
            d_steering = (steering_percentage - self.last_steering_percentage) \
                / (timestamp_sec - self.last_timestamp_sec)
            self.speed_data.append(speed_mps)
            self.d_steering_data.append(d_steering)

            self.last_steering_percentage = steering_percentage
            self.last_speed_mps = speed_mps
            self.last_timestamp_sec = timestamp_sec

    def get_speed_dsteering(self):
        return self.speed_data, self.d_steering_data

if __name__ == "__main__":
    import sys
    import matplotlib.pyplot as plt
    from os import listdir
    from os.path import isfile, join

    folders = sys.argv[1:]
    fig, ax = plt.subplots()
    colors = ["g", "b", "r", "m", "y"]
    markers = ["o", "o", "o", "o"]
    for i in range(len(folders)):
        folder = folders[i]
        color = colors[i % len(colors)]
        marker = markers[i % len(markers)]
        fns = [f for f in listdir(folder) if isfile(join(folder, f))]
        for fn in fns:
            reader = RecordItemReader(folder+"/"+fn)
            processor = SpeedDsteeringData()
            last_pose_data = None
            last_chassis_data = None
            topics = ["/apollo/localization/pose", "/apollo/canbus/chassis"]
            for data in reader.read(topics):
                if "chassis" in data:
                    last_chassis_data = data["chassis"]
                if last_chassis_data is not None:
                    processor.add(last_chassis_data)
                    #last_pose_data = None
                    #last_chassis_data = None

            data_x, data_y = processor.get_speed_dsteering()
            ax.scatter(data_x, data_y, c=color, marker=marker, alpha=0.2)

    plt.show()