import pandas as pd
import numpy as np
from fitensemble import bayesian_weighting, belt
import experiment_loader
import ALA3

prior = "BW"
ff = "amber96"
stride = 1000
regularization_strength = 10.0

thin = 400
factor = 50
steps = 1000000

predictions_framewise, measurements, uncertainties = experiment_loader.load(ff, stride=stride)
phi, psi, ass_raw0, state_ind0 = experiment_loader.load_rama(ff, stride)

num_states = len(phi)
assignments = np.arange(num_states)

prior_pops = np.ones(num_states)

predictions = pd.DataFrame(bayesian_weighting.framewise_to_statewise(predictions_framewise, assignments), columns=predictions_framewise.columns)
model = bayesian_weighting.MaxentBayesianWeighting(predictions.values, measurements.values, uncertainties.values, assignments, regularization_strength)
model.sample(steps * factor, thin=thin * factor)

model2 = belt.MaxEntBELT(predictions.values, measurements.values, uncertainties.values, regularization_strength)
model2.sample(steps, thin=thin)

pi = model.mcmc.trace("matrix_populations")[:, 0]

num_samples = len(pi)

data = np.zeros((num_samples, num_samples))
for i, p in enumerate(model.iterate_populations()):
    print(i)
    for j, p2 in enumerate(model2.iterate_populations()):
        data[i, j] = p.dot(np.log(p / p2))

p_bw = model.accumulate_populations()
p_BELT = model2.accumulate_populations()

chi2 = []
prior = []
H_terms = []
for j, p2 in enumerate(model2.iterate_populations()):
    mu = predictions.T.dot(p2)
    chi2.append(0.5 * (((mu - measurements) / uncertainties) ** 2).sum())
    prior.append(regularization_strength * -1.0 * p2.dot(np.log(p2)))
    H = -np.diag(p2[:-1] ** -1.) - p[-1] ** -1.
    H_terms.append(0.5 * np.linalg.slogdet(H)[1])

R = pd.DataFrame({"chi2":chi2, "prior":prior, "H":H_terms})


# -*- coding: utf-8 -*-
#
# brette_gerstner_fig_3d.py
#
# This file is part of NEST.
#
# Copyright (C) 2004 The NEST Initiative
#
# NEST is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# NEST is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with NEST.  If not, see <http://www.gnu.org/licenses/>.

"""Testing the adapting exponential integrate and fire model in NEST (Brette and Gerstner Fig 3D)
----------------------------------------------------------------------------------------------------

This example tests the adaptive integrate and fire model (AdEx) according to
Brette and Gerstner [1]_ reproduces Figure 3D of the paper.

Note that Brette and Gerstner give the value for `b` in `nA`.
To be consistent with the other parameters in the equations, `b` must be
converted to `pA` (pico Ampere).

References
~~~~~~~~~~~

.. [1] Brette R and Gerstner W (2005). Adaptive exponential integrate-and-fire model as an effective
       description of neuronal activity J. Neurophysiology. https://doi.org/10.1152/jn.00686.2005
"""

import nest
import nest.voltage_trace
import matplotlib.pyplot as plt

nest.ResetKernel()

###############################################################################
# First we make sure that the resolution of the simulation is 0.1 ms. This is
# important, since the slop of the action potential is very steep.

res = 0.1
nest.SetKernelStatus({"resolution": res})
neuron = nest.Create("aeif_cond_exp")

###############################################################################
# Set the parameters of the neuron according to the paper.

neuron.set(V_peak=20., E_L=-60.0, a=80.0, b=80.5, tau_w=720.0)

###############################################################################
# Create and configure the stimulus which is a step current.

dc = nest.Create("dc_generator")

dc.set(amplitude=-800.0, start=0.0, stop=400.0)

###############################################################################
# We connect the DC generators.

nest.Connect(dc, neuron, 'all_to_all')

###############################################################################
# And add a ``voltmeter`` to sample the membrane potentials from the neuron
# in intervals of 0.1 ms.

voltmeter = nest.Create("voltmeter", params={'interval': 0.1})
nest.Connect(voltmeter, neuron)

###############################################################################
# Finally, we simulate for 1000 ms and plot a voltage trace to produce the
# figure.

nest.Simulate(1000.0)

nest.voltage_trace.from_device(voltmeter)
plt.axis([0, 1000, -85, 0])
nest.voltage_trace.show()


from odm2api.models import Base, TimeSeriesResultValues
from sqlalchemy.exc import IntegrityError, ProgrammingError
import sqlalchemy.ext.declarative.api as api
import pandas as pd
from sqlalchemy import func

class iOutputs:

    def __init__(self):
        pass

    def parseObjects(self, session):
        data = {}

        schema = TimeSeriesResultValues.__table_args__['schema']

        for t in self.get_table_names():

            tmplist = []
            try:
                if t.__tablename__.lower() == "timeseriesresultvalues":
                    # TODO: Test if this works for database connections to mssql and mysql
                    if 'postgresql' in session.bind.name:
                        sql = """SELECT * FROM {}.timeseriesresultvalues""".format(schema)
                    elif 'mssql' in session.bind.name:
                        sql = """SELECT * FROM {}.TimeSeriesResultValues""".format(schema)
                    else:
                        sql = """SELECT * FROM TimeSeriesResultValues"""
                    tbl = pd.read_sql(sql, session.connection().connection.connection)
                    tmplist = tbl
                else:

                    try:
                        for obj in session.query(t).all():
                            # session.expunge(o)
                            tmplist.append(obj)

                    except ProgrammingError as e:

                        print(e.message)

            except IntegrityError as e:
                print(e)
                session.rollback()

            if len(tmplist) > 0:
                data[t.__tablename__] = tmplist

        return data

    def get_table_names(self):
        tables = []
        import inspect
        import sys
        # get a list of all of the classes in the module
        clsmembers = inspect.getmembers(sys.modules["odm2api.models"],
                                        lambda member: inspect.isclass(member) and member.__module__ == "odm2api.models")

        for name, Tbl in clsmembers:
            if isinstance(Tbl, api.DeclarativeMeta):
                # check to see if the schema is already set correctly
                tables.append(Tbl)

        return tables

    def save(self, session, path):
        raise NotImplementedError()

    def accept(self):
        raise NotImplementedError()


import os,shutil
from glob import iglob

import matplotlib.pyplot as plt
from chxanalys.chx_libs import (np, roi, time, datetime, os,  getpass, db,
                                      get_images,LogNorm, RUN_GUI)
from chxanalys.chx_generic_functions import (create_time_slice,get_detector, get_fields, get_sid_filenames,
     load_data)

import struct
from tqdm import tqdm
from contextlib import closing

from multiprocessing import Pool
import dill
import sys
import gc
import pickle as pkl
from eiger_io.pims_reader import EigerImages

def run_dill_encoded(what):
    fun, args = dill.loads(what)
    return fun(*args)

def apply_async(pool, fun, args, callback=None):
    return pool.apply_async( run_dill_encoded, (dill.dumps((fun, args)),), callback= callback)

def map_async(pool, fun, args ):
    return pool.map_async(run_dill_encoded,  (dill.dumps((fun, args)),))

def pass_FD(FD,n):
    #FD.rdframe(n)
    FD.seekimg(n)

def go_through_FD(FD):
    for i in range(FD.beg, FD.end):
        pass_FD(FD,i)

def compress_eigerdata( images, mask, md, filename=None,  force_compress=False,
                        bad_pixel_threshold=1e15, bad_pixel_low_threshold=0,
                       hot_pixel_threshold=2**30, nobytes=4,bins=1, bad_frame_list=None,
                       para_compress= False, num_sub=100, dtypes='uid',reverse =True,
                      num_max_para_process=500, with_pickle=False, direct_load_data=False, data_path=None):

    end= len(images)//bins

    if filename is None:
        filename=  '/XF11ID/analysis/Compressed_Data' +'/uid_%s.cmp'%md['uid']

    if dtypes!= 'uid':
        para_compress= False
    else:
        if para_compress:
            images='foo'
            #para_compress=   True

    #print( dtypes )

    if force_compress:
        print ("Create a new compress file with filename as :%s."%filename)
        if para_compress:
            print( 'Using a multiprocess to compress the data.')
            return para_compress_eigerdata( images, mask, md, filename,
                        bad_pixel_threshold=bad_pixel_threshold, hot_pixel_threshold=hot_pixel_threshold,
                                    bad_pixel_low_threshold=bad_pixel_low_threshold,nobytes= nobytes, bins=bins,
                                     num_sub=num_sub, dtypes=dtypes, reverse=reverse,
                                          num_max_para_process=num_max_para_process, with_pickle= with_pickle,
                                           direct_load_data= direct_load_data,data_path=data_path)

        else:
            return init_compress_eigerdata( images, mask, md, filename,
                        bad_pixel_threshold=bad_pixel_threshold, hot_pixel_threshold=hot_pixel_threshold,
                                    bad_pixel_low_threshold=bad_pixel_low_threshold,nobytes= nobytes, bins=bins,with_pickle= with_pickle, direct_load_data= direct_load_data,data_path=data_path )
    else:
        if not os.path.exists( filename ):
            print ("Create a new compress file with filename as :%s."%filename)
            if para_compress:
                print( 'Using a multiprocess to compress the data.')
                return para_compress_eigerdata( images, mask, md, filename,
                        bad_pixel_threshold=bad_pixel_threshold, hot_pixel_threshold=hot_pixel_threshold,
                                    bad_pixel_low_threshold=bad_pixel_low_threshold,nobytes= nobytes, bins=bins,
                                     num_sub=num_sub, dtypes=dtypes, reverse=reverse,
                                              num_max_para_process=num_max_para_process,with_pickle= with_pickle, direct_load_data= direct_load_data,data_path=data_path)
            else:
                return init_compress_eigerdata( images, mask, md, filename,
                       bad_pixel_threshold=bad_pixel_threshold, hot_pixel_threshold=hot_pixel_threshold,
                      bad_pixel_low_threshold=bad_pixel_low_threshold,    nobytes= nobytes, bins=bins,with_pickle= with_pickle, direct_load_data= direct_load_data,data_path=data_path  )
        else:
            print ("Using already created compressed file with filename as :%s."%filename)
            beg=0
            return read_compressed_eigerdata( mask, filename, beg, end,
                            bad_pixel_threshold=bad_pixel_threshold, hot_pixel_threshold=hot_pixel_threshold,
                       bad_pixel_low_threshold=bad_pixel_low_threshold ,bad_frame_list=bad_frame_list,with_pickle= with_pickle, direct_load_data= direct_load_data,data_path=data_path    )

def read_compressed_eigerdata( mask, filename, beg, end,
                              bad_pixel_threshold=1e15, hot_pixel_threshold=2**30,
                             bad_pixel_low_threshold=0,bad_frame_list=None,with_pickle= False,
                             direct_load_data=False,data_path=None):
    '''
        Read already compress eiger data
        Return
            mask
            avg_img
            imsum
            bad_frame_list

    '''
    #should use try and except instead of with_pickle in the future!
    CAL = False
    if not with_pickle:
        CAL = True
    else:
        try:
            mask, avg_img, imgsum, bad_frame_list_ =  pkl.load( open(filename + '.pkl', 'rb' ) )
        except:
            CAL = True
    if CAL:
        FD = Multifile( filename, beg, end)
        imgsum  =  np.zeros(   FD.end- FD.beg, dtype= np.float  )
        avg_img = np.zeros(  [FD.md['ncols'], FD.md['nrows'] ] , dtype= np.float )
        imgsum, bad_frame_list_ = get_each_frame_intensityc( FD, sampling = 1,
                bad_pixel_threshold=bad_pixel_threshold, bad_pixel_low_threshold=bad_pixel_low_threshold,
                                        hot_pixel_threshold=hot_pixel_threshold, plot_ = False,
                                                bad_frame_list=bad_frame_list)
        avg_img = get_avg_imgc( FD,  beg=None,end=None,sampling = 1, plot_ = False,bad_frame_list=bad_frame_list_ )
        FD.FID.close()

    return   mask, avg_img, imgsum, bad_frame_list_

def para_compress_eigerdata(  images, mask, md, filename, num_sub=100,
                        bad_pixel_threshold=1e15, hot_pixel_threshold=2**30,
                            bad_pixel_low_threshold=0, nobytes=4, bins=1, dtypes='uid',reverse =True,
                           num_max_para_process=500, cpu_core_number=72, with_pickle=True,
                           direct_load_data=False, data_path=None):

    if dtypes=='uid':
        uid= md['uid'] #images
        if not direct_load_data:
            detector = get_detector( db[uid ] )
            images_ = load_data( uid, detector, reverse= reverse    )
        else:
            images_ = EigerImages(data_path, md)
        N= len(images_)

    else:
        N = len(images)
    N = int( np.ceil( N/ bins  ) )
    Nf  =  int( np.ceil( N/ num_sub  ) )
    if Nf >   cpu_core_number:
        print("The process number is larger than %s (XF11ID server core number)"%cpu_core_number)
        num_sub_old = num_sub
        num_sub = int( np.ceil(N/cpu_core_number))
        Nf  =  int( np.ceil( N/ num_sub  ) )
        print ("The sub compressed file number was changed from %s to %s"%( num_sub_old, num_sub ))
    create_compress_header( md, filename +'-header', nobytes, bins  )
    #print( 'done for header here')
    results = para_segment_compress_eigerdata( images=images, mask=mask, md=md,filename=filename,
                    num_sub=num_sub, bad_pixel_threshold=bad_pixel_threshold, hot_pixel_threshold=hot_pixel_threshold,
                    bad_pixel_low_threshold=bad_pixel_low_threshold,nobytes=nobytes, bins=bins, dtypes=dtypes,
                                             num_max_para_process=num_max_para_process,
                                            direct_load_data=direct_load_data, data_path=data_path)

    res_ = np.array( [ results[k].get() for k in   list(sorted(results.keys()))   ]     )
    imgsum = np.zeros( N )
    bad_frame_list = np.zeros( N, dtype=bool )
    good_count = 1
    for i in range( Nf ):
        mask_, avg_img_, imgsum_, bad_frame_list_ = res_[i]
        imgsum[i*num_sub: (i+1)*num_sub] = imgsum_
        bad_frame_list[i*num_sub: (i+1)*num_sub] = bad_frame_list_
        if i==0:
            mask = mask_
            avg_img = np.zeros_like( avg_img_ )
        else:
            mask *= mask_
        if not np.sum( np.isnan( avg_img_)):
            avg_img += avg_img_
            good_count += 1

    bad_frame_list = np.where(  bad_frame_list )[0]
    avg_img /= good_count

    if len(bad_frame_list):
        print ('Bad frame list are: %s' %bad_frame_list)
    else:
        print ('No bad frames are involved.')
    print( 'Combining the seperated compressed files together...')
    combine_compressed( filename, Nf, del_old=True)

    del results
    del res_
    if  with_pickle:
        pkl.dump( [mask, avg_img, imgsum, bad_frame_list], open(filename + '.pkl', 'wb' ) )
    return   mask, avg_img, imgsum, bad_frame_list

def combine_compressed( filename,  Nf, del_old=True):
    old_files = np.concatenate(  np.array([ [filename +'-header'],
                                            [filename  + '_temp-%i.tmp'%i for i in range(Nf) ]]))
    combine_binary_files(filename, old_files, del_old  )

def  combine_binary_files(filename, old_files, del_old = False):
    '''Combine binary files together'''
    fn_ = open(filename, 'wb')
    for ftemp in old_files:
        shutil.copyfileobj( open(ftemp, 'rb'), fn_)
        if del_old:
            os.remove( ftemp )
    fn_.close()

def para_segment_compress_eigerdata( images, mask,  md, filename, num_sub=100,
                        bad_pixel_threshold=1e15, hot_pixel_threshold=2**30,
                            bad_pixel_low_threshold=0, nobytes=4, bins=1,  dtypes='images',reverse =True,
                                   num_max_para_process=50,direct_load_data=False, data_path=None):
    '''
    parallelly compressed eiger data without header, this function is for parallel compress
    '''
    if dtypes=='uid':
        uid= md['uid'] #images
        if not direct_load_data:
            detector = get_detector( db[uid ] )
            images_ = load_data( uid, detector, reverse= reverse    )
        else:
            images_ = EigerImages(data_path, md)
        N= len(images_)

    else:
        N = len(images)

    #N = int( np.ceil( N/ bins  ) )
    num_sub *= bins
    if N%num_sub:
        Nf = N// num_sub +1
        print('The average image intensity would be slightly not correct, about 1% error.')
        print( 'Please give a num_sub to make reminder of Num_images/num_sub =0 to get a correct avg_image')
    else:
        Nf = N//num_sub
    print( 'It will create %i temporary files for parallel compression.'%Nf)

    if Nf> num_max_para_process:
        N_runs = np.int( np.ceil( Nf/float(num_max_para_process)))
        print('The parallel run number: %s is larger than num_max_para_process: %s'%(Nf, num_max_para_process ))
    else:
        N_runs= 1
    result = {}
    #print( mask_filename )# + '*'* 10 + 'here' )
    for nr in range( N_runs ):
        if (nr+1)*num_max_para_process > Nf:
            inputs= range( num_max_para_process*nr, Nf )
        else:
            inputs= range( num_max_para_process*nr, num_max_para_process*(nr + 1 ) )
        fns = [  filename + '_temp-%i.tmp'%i for i in inputs]
        #print( nr, inputs, )
        pool =  Pool(processes= len(inputs) ) #, maxtasksperchild=1000 )
        #print( inputs )
        for i in  inputs:
            if i*num_sub <= N:
                result[i] = pool.apply_async(  segment_compress_eigerdata, [
                    images,  mask,  md,  filename + '_temp-%i.tmp'%i,bad_pixel_threshold, hot_pixel_threshold,    bad_pixel_low_threshold, nobytes, bins, i*num_sub, (i+1)*num_sub, dtypes, reverse,direct_load_data, data_path  ]  )

        pool.close()
        pool.join()
        pool.terminate()
    return result

def segment_compress_eigerdata( images,  mask, md, filename,
                        bad_pixel_threshold=1e15, hot_pixel_threshold=2**30,
                            bad_pixel_low_threshold=0, nobytes=4, bins=1,
                               N1=None, N2=None, dtypes='images',reverse =True,direct_load_data=False, data_path=None    ):
    '''
    Create a compressed eiger data without header, this function is for parallel compress
    for parallel compress don't pass any non-scalar parameters
    '''

    if dtypes=='uid':
        uid= md['uid'] #images
        if not direct_load_data:
            detector = get_detector( db[uid ] )
            images = load_data( uid, detector, reverse= reverse    )[N1:N2]
        else:
            images = EigerImages(data_path, md)[N1:N2]
    Nimg_ = len( images)
    M,N = images[0].shape
    avg_img = np.zeros( [M,N], dtype= np.float )
    Nopix =  float( avg_img.size )
    n=0
    good_count = 0
    #frac = 0.0
    if nobytes==2:
        dtype= np.int16
    elif nobytes==4:
        dtype= np.int32
    elif nobytes==8:
        dtype=np.float64
    else:
        print ( "Wrong type of nobytes, only support 2 [np.int16] or 4 [np.int32]")
        dtype= np.int32

    #Nimg =   Nimg_//bins
    Nimg = int( np.ceil( Nimg_ / bins  ) )
    time_edge = np.array(create_time_slice( N= Nimg_,
                                    slice_num= Nimg, slice_width= bins ))
    #print( time_edge, Nimg_, Nimg, bins, N1, N2 )
    imgsum  =  np.zeros(    Nimg   )
    if bins!=1:
        #print('The frames will be binned by %s'%bins)
        dtype=np.float64

    fp = open( filename,'wb' )
    for n in   range(Nimg):
        t1,t2 = time_edge[n]
        if bins!=1:
            img = np.array( np.average(  images[t1:t2], axis=0   )   , dtype= dtype)
        else:
            img =   np.array( images[t1], dtype=dtype)
        mask &= img < hot_pixel_threshold
        p = np.where( (np.ravel(img)>0) *  np.ravel(mask) )[0] #don't use masked data
        v = np.ravel( np.array( img, dtype= dtype )) [p]
        dlen = len(p)
        imgsum[n] = v.sum()
        if (dlen==0) or (imgsum[n] > bad_pixel_threshold) or (imgsum[n] <=bad_pixel_low_threshold):
            dlen = 0
            fp.write(  struct.pack( '@I', dlen  ))
        else:
            np.ravel( avg_img )[p] +=   v
            good_count +=1
            fp.write(  struct.pack( '@I', dlen   ))
            fp.write(  struct.pack( '@{}i'.format( dlen), *p))
            if bins==1:
                fp.write(  struct.pack( '@{}{}'.format( dlen,'ih'[nobytes==2]), *v))
            else:
                fp.write(  struct.pack( '@{}{}'.format( dlen,'dd'[nobytes==2]  ), *v))        #n +=1
        del  p,v, img
        fp.flush()
    fp.close()
    avg_img /= good_count
    bad_frame_list =  (np.array(imgsum) > bad_pixel_threshold) | (np.array(imgsum) <= bad_pixel_low_threshold)
    sys.stdout.write('#')
    sys.stdout.flush()
    #del  images, mask, avg_img, imgsum, bad_frame_list
    #print( 'Should release memory here')
    return   mask, avg_img, imgsum, bad_frame_list

def create_compress_header( md, filename, nobytes=4, bins=1  ):
    '''
    Create the head for a compressed eiger data, this function is for parallel compress
    '''
    fp = open( filename,'wb' )
    #Make Header 1024 bytes
    #md = images.md
    if bins!=1:
        nobytes=8

    Header = struct.pack('@16s8d7I916x',b'Version-COMP0001',
                        md['beam_center_x'],md['beam_center_y'], md['count_time'], md['detector_distance'],
                        md['frame_time'],md['incident_wavelength'], md['x_pixel_size'],md['y_pixel_size'],
                        nobytes, md['pixel_mask'].shape[1], md['pixel_mask'].shape[0],
                         0, md['pixel_mask'].shape[1],
                         0, md['pixel_mask'].shape[0]
                    )

    fp.write( Header)
    fp.close()

def init_compress_eigerdata( images, mask, md, filename,
                        bad_pixel_threshold=1e15, hot_pixel_threshold=2**30,
                            bad_pixel_low_threshold=0,nobytes=4, bins=1, with_pickle=True,
                           direct_load_data=False, data_path=None):
    '''
        Compress the eiger data

        Create a new mask by remove hot_pixel
        Do image average
        Do each image sum
        Find badframe_list for where image sum above bad_pixel_threshold
        Generate a compressed data with filename

        if bins!=1, will bin the images with bin number as bins

        Header contains 1024 bytes ['Magic value', 'beam_center_x', 'beam_center_y', 'count_time', 'detector_distance',
           'frame_time', 'incident_wavelength', 'x_pixel_size', 'y_pixel_size',
           bytes per pixel (either 2 or 4 (Default)),
           Nrows, Ncols, Rows_Begin, Rows_End, Cols_Begin, Cols_End ]

        Return
            mask
            avg_img
            imsum
            bad_frame_list

    '''
    fp = open( filename,'wb' )
    #Make Header 1024 bytes
    #md = images.md
    if bins!=1:
        nobytes=8

    Header = struct.pack('@16s8d7I916x',b'Version-COMP0001',
                        md['beam_center_x'],md['beam_center_y'], md['count_time'], md['detector_distance'],
                        md['frame_time'],md['incident_wavelength'], md['x_pixel_size'],md['y_pixel_size'],
                        nobytes, md['pixel_mask'].shape[1], md['pixel_mask'].shape[0],
                         0, md['pixel_mask'].shape[1],
                         0, md['pixel_mask'].shape[0]
                    )

    fp.write( Header)

    Nimg_ = len( images)
    avg_img = np.zeros_like(    images[0], dtype= np.float )
    Nopix =  float( avg_img.size )
    n=0
    good_count = 0
    frac = 0.0
    if nobytes==2:
        dtype= np.int16
    elif nobytes==4:
        dtype= np.int32
    elif nobytes==8:
        dtype=np.float64
    else:
        print ( "Wrong type of nobytes, only support 2 [np.int16] or 4 [np.int32]")
        dtype= np.int32

    Nimg =   Nimg_//bins
    time_edge = np.array(create_time_slice( N= Nimg_,
                                    slice_num= Nimg, slice_width= bins ))

    imgsum  =  np.zeros(    Nimg   )
    if bins!=1:
        print('The frames will be binned by %s'%bins)

    for n in  tqdm( range(Nimg) ):
        t1,t2 = time_edge[n]
        img = np.average(  images[t1:t2], axis=0   )
        mask &= img < hot_pixel_threshold
        p = np.where( (np.ravel(img)>0) &  np.ravel(mask) )[0] #don't use masked data
        v = np.ravel( np.array( img, dtype= dtype )) [p]
        dlen = len(p)
        imgsum[n] = v.sum()
        if (imgsum[n] >bad_pixel_threshold) or (imgsum[n] <=bad_pixel_low_threshold):
        #if imgsum[n] >=bad_pixel_threshold :
            dlen = 0
            fp.write(  struct.pack( '@I', dlen  ))
        else:
            np.ravel(avg_img )[p] +=   v
            good_count +=1
            frac += dlen/Nopix
            #s_fmt ='@I{}i{}{}'.format( dlen,dlen,'ih'[nobytes==2])
            fp.write(  struct.pack( '@I', dlen   ))
            fp.write(  struct.pack( '@{}i'.format( dlen), *p))
            if bins==1:
                fp.write(  struct.pack( '@{}{}'.format( dlen,'ih'[nobytes==2]), *v))
            else:
                fp.write(  struct.pack( '@{}{}'.format( dlen,'dd'[nobytes==2]  ), *v))
        #n +=1

    fp.close()
    frac /=good_count
    print( "The fraction of pixel occupied by photon is %6.3f%% "%(100*frac) )
    avg_img /= good_count

    bad_frame_list = np.where( (np.array(imgsum) > bad_pixel_threshold) | (np.array(imgsum) <= bad_pixel_low_threshold)  )[0]
    #bad_frame_list1 = np.where( np.array(imgsum) > bad_pixel_threshold  )[0]
    #bad_frame_list2 = np.where( np.array(imgsum) < bad_pixel_low_threshold  )[0]
    #bad_frame_list =   np.unique( np.concatenate( [bad_frame_list1, bad_frame_list2]) )

    if len(bad_frame_list):
        print ('Bad frame list are: %s' %bad_frame_list)
    else:
        print ('No bad frames are involved.')
    if  with_pickle:
        pkl.dump( [mask, avg_img, imgsum, bad_frame_list], open(filename + '.pkl', 'wb' ) )
    return   mask, avg_img, imgsum, bad_frame_list

"""    Description:

    This is code that Mark wrote to open the multifile format
    in compressed mode, translated to python.
    This seems to work for DALSA, FCCD and EIGER in compressed mode.
    It should be included in the respective detector.i files
    Currently, this refers to the compression mode being '6'
    Each file is image descriptor files chunked together as follows:
            Header (1024 bytes)
    |--------------IMG N begin--------------|
    |                   Dlen
    |---------------------------------------|
    |       Pixel positions (dlen*4 bytes   |
    |      (0 based indexing in file)       |
    |---------------------------------------|
    |    Pixel data(dlen*bytes bytes)       |
    |    (bytes is found in header          |
    |    at position 116)                   |
    |--------------IMG N end----------------|
    |--------------IMG N+1 begin------------|
    |----------------etc.....---------------|

     Header contains 1024 bytes version name, 'beam_center_x', 'beam_center_y', 'count_time', 'detector_distance',
           'frame_time', 'incident_wavelength', 'x_pixel_size', 'y_pixel_size',
           bytes per pixel (either 2 or 4 (Default)),
           Nrows, Ncols, Rows_Begin, Rows_End, Cols_Begin, Cols_End,

"""

class Multifile:
    '''The class representing the multifile.
        The recno is in 1 based numbering scheme (first record is 1)
	This is efficient for reading in increasing order.
	Note: reading same image twice in a row is like reading an earlier
	numbered image and means the program starts for the beginning again.

    '''
    def __init__(self,filename,beg,end):
        '''Multifile initialization. Open the file.
            Here I use the read routine which returns byte objects
            (everything is an object in python). I use struct.unpack
            to convert the byte object to other data type (int object
            etc)
            NOTE: At each record n, the file cursor points to record n+1
        '''
        self.FID = open(filename,"rb")
#        self.FID.seek(0,os.SEEK_SET)
        self.filename = filename
        #br: bytes read
        br = self.FID.read(1024)
        self.beg=beg
        self.end=end
        ms_keys = ['beam_center_x', 'beam_center_y', 'count_time', 'detector_distance',
           'frame_time', 'incident_wavelength', 'x_pixel_size', 'y_pixel_size',
           'bytes',
            'nrows', 'ncols', 'rows_begin', 'rows_end', 'cols_begin', 'cols_end'
          ]

        magic = struct.unpack('@16s', br[:16])
        md_temp =  struct.unpack('@8d7I916x', br[16:])
        self.md = dict(zip(ms_keys, md_temp))

        self.imgread=0
        self.recno = 0
        # some initialization stuff
        self.byts = self.md['bytes']
        if (self.byts==2):
            self.valtype = np.uint16
        elif (self.byts == 4):
            self.valtype = np.uint32
        elif (self.byts == 8):
            self.valtype = np.float64
        #now convert pieces of these bytes to our data
        self.dlen =np.fromfile(self.FID,dtype=np.int32,count=1)[0]

        # now read first image
        #print "Opened file. Bytes per data is {0img.shape = (self.rows,self.cols)}".format(self.byts)

    def _readHeader(self):
        self.dlen =np.fromfile(self.FID,dtype=np.int32,count=1)[0]

    def _readImageRaw(self):

        p= np.fromfile(self.FID, dtype = np.int32,count= self.dlen)
        v= np.fromfile(self.FID, dtype = self.valtype,count= self.dlen)
        self.imgread=1
        return(p,v)

    def _readImage(self):
        (p,v)=self._readImageRaw()
        img = np.zeros( (  self.md['ncols'], self.md['nrows']   ) )
        np.put( np.ravel(img), p, v )
        return(img)

    def seekimg(self,n=None):

        '''Position file to read the nth image.
            For now only reads first image ignores n
        '''
        # the logic involving finding the cursor position
        if (n is None):
            n = self.recno
        if (n < self.beg or n > self.end):
            raise IndexError('Error, record out of range')
        #print (n, self.recno, self.FID.tell() )
        if ((n == self.recno)  and (self.imgread==0)):
            pass # do nothing

        else:
            if (n <= self.recno): #ensure cursor less than search pos
                self.FID.seek(1024,os.SEEK_SET)
                self.dlen =np.fromfile(self.FID,dtype=np.int32,count=1)[0]
                self.recno = 0
                self.imgread=0
                if n == 0:
                    return
            #have to iterate on seeking since dlen varies
            #remember for rec recno, cursor is always at recno+1
            if(self.imgread==0 ): #move to next header if need to
                self.FID.seek(self.dlen*(4+self.byts),os.SEEK_CUR)
            for i in range(self.recno+1,n):
                #the less seeks performed the faster
                #print (i)
                self.dlen =np.fromfile(self.FID,dtype=np.int32,count=1)[0]
                #print 's',self.dlen
                self.FID.seek(self.dlen*(4+self.byts),os.SEEK_CUR)

            # we are now at recno in file, read the header and data
            #self._clearImage()
            self._readHeader()
            self.imgread=0
            self.recno = n
    def rdframe(self,n):
        if self.seekimg(n)!=-1:
            return(self._readImage())

    def rdrawframe(self,n):
        if self.seekimg(n)!=-1:
             return(self._readImageRaw())

def pass_FD(FD,n):
    #FD.rdframe(n)
    FD.seekimg(n)

class Multifile_Bins( object  ):
    '''
    Bin a compressed file with bins number
    See Multifile for details for Multifile_class
    '''
    def __init__(self, FD, bins=100):
        '''
        FD: the handler of a compressed Eiger frames
        bins: bins number
       '''

        self.FD=FD
        if (FD.end - FD.beg)%bins:
            print ('Please give a better bins number and make the length of FD/bins= integer')
        else:
            self.bins = bins
            self.md = FD.md
            #self.beg = FD.beg
            self.beg = 0
            Nimg = (FD.end - FD.beg)
            slice_num =   Nimg//bins
            self.end =  slice_num
            self.time_edge = np.array(create_time_slice( N= Nimg,
                                    slice_num= slice_num, slice_width= bins )) + FD.beg
            self.get_bin_frame()

    def get_bin_frame(self):
        FD= self.FD
        self.frames = np.zeros( [ FD.md['ncols'],FD.md['nrows'], len(self.time_edge)] )
        for n in  tqdm( range(len(self.time_edge))):
            #print (n)
            t1,t2 = self.time_edge[n]
            #print( t1, t2)
            self.frames[:,:,n] = get_avg_imgc( FD, beg=t1,end=t2, sampling = 1,
                                              plot_ = False, show_progress = False )
    def rdframe(self,n):
        return self.frames[:,:,n]

    def rdrawframe(self,n):
        x_= np.ravel(  self.rdframe(n) )
        p=  np.where( x_ ) [0]
        v =  np.array( x_[ p ])
        return ( np.array(p, dtype=np.int32), v)

def get_avg_imgc( FD,  beg=None,end=None, sampling = 100, plot_ = False, bad_frame_list=None,
                 show_progress=True, *argv,**kwargs):
    '''Get average imagef from a data_series by every sampling number to save time'''
    #avg_img = np.average(data_series[:: sampling], axis=0)

    if beg is None:
        beg = FD.beg
    if end is None:
        end = FD.end

    avg_img = FD.rdframe(beg)
    n=1
    flag=True
    if show_progress:
        #print(  sampling-1 + beg , end, sampling )
        if bad_frame_list is None:
            bad_frame_list =[]
        fra_num = int( (end -  beg )/sampling ) - len( bad_frame_list )
        for  i in tqdm(range( sampling-1 + beg , end, sampling  ), desc= 'Averaging %s images'% fra_num):
            if bad_frame_list is not None:
                if i in bad_frame_list:
                    flag= False
                else:
                    flag=True
            #print(i, flag)
            if flag:
                (p,v) = FD.rdrawframe(i)
                if len(p)>0:
                    np.ravel(avg_img )[p] +=   v
                    n += 1
    else:
        for  i in range( sampling-1 + beg , end, sampling  ):
            if bad_frame_list is not None:
                if i in bad_frame_list:
                    flag= False
                else:
                    flag=True
            if flag:
                (p,v) = FD.rdrawframe(i)
                if len(p)>0:
                    np.ravel(avg_img )[p] +=   v
                    n += 1

    avg_img /= n
    if plot_:
        if RUN_GUI:
            fig = Figure()
            ax = fig.add_subplot(111)
        else:
            fig, ax = plt.subplots()
        uid = 'uid'
        if 'uid' in kwargs.keys():
            uid = kwargs['uid']
        im = ax.imshow(avg_img , cmap='viridis',origin='lower',
                   norm= LogNorm(vmin=0.001, vmax=1e2))
        #ax.set_title("Masked Averaged Image")
        ax.set_title('uid= %s--Masked-Averaged-Image-'%uid)
        fig.colorbar(im)
        if save:
            #dt =datetime.now()
            #CurTime = '%s%02d%02d-%02d%02d-' % (dt.year, dt.month, dt.day,dt.hour,dt.minute)
            path = kwargs['path']
            if 'uid' in kwargs:
                uid = kwargs['uid']
            else:
                uid = 'uid'
            #fp = path + "uid= %s--Waterfall-"%uid + CurTime + '.png'
            fp = path + "uid=%s--avg-img-"%uid  + '.png'
            plt.savefig( fp, dpi=fig.dpi)
        #plt.show()
    return avg_img

def mean_intensityc(FD, labeled_array,  sampling=1, index=None, multi_cor = False):
    """Compute the mean intensity for each ROI in the compressed file (FD), support parallel computation

    Parameters
    ----------
    FD: Multifile class
        compressed file
    labeled_array : array
        labeled array; 0 is background.
        Each ROI is represented by a nonzero integer. It is not required that
        the ROI labels are contiguous
    index : int, list, optional
        The ROI's to use. If None, this function will extract averages for all
        ROIs

    Returns
    -------
    mean_intensity : array
        The mean intensity of each ROI for all `images`
        Dimensions:
            len(mean_intensity) == len(index)
            len(mean_intensity[0]) == len(images)
    index : list
        The labels for each element of the `mean_intensity` list
    """

    qind, pixelist = roi.extract_label_indices(  labeled_array  )

    if labeled_array.shape != ( FD.md['ncols'],FD.md['nrows']):
        raise ValueError(
            " `image` shape (%d, %d) in FD is not equal to the labeled_array shape (%d, %d)" %( FD.md['ncols'],FD.md['nrows'], labeled_array.shape[0], labeled_array.shape[1]) )
    # handle various input for `index`
    if index is None:
        index = list(np.unique(labeled_array))
        index.remove(0)
    else:
        try:
            len(index)
        except TypeError:
            index = [index]

        index = np.array( index )
        #print ('here')
        good_ind  = np.zeros( max(qind), dtype= np.int32 )
        good_ind[ index -1  ]   =    np.arange( len(index) ) +1
        w =  np.where( good_ind[qind -1 ]   )[0]
        qind = good_ind[ qind[w] -1 ]
        pixelist = pixelist[w]

    # pre-allocate an array for performance
    # might be able to use list comprehension to make this faster

    mean_intensity = np.zeros(   [ int( ( FD.end - FD.beg)/sampling ) , len(index)] )
    #fra_pix = np.zeros_like( pixelist, dtype=np.float64)
    timg = np.zeros(    FD.md['ncols'] * FD.md['nrows']   , dtype=np.int32   )
    timg[pixelist] =   np.arange( 1, len(pixelist) + 1  )
    #maxqind = max(qind)
    norm = np.bincount( qind  )[1:]
    n= 0
    #for  i in tqdm(range( FD.beg , FD.end )):
    if not multi_cor:
        for  i in tqdm(range( FD.beg, FD.end, sampling  ), desc= 'Get ROI intensity of each frame' ):
            (p,v) = FD.rdrawframe(i)
            w = np.where( timg[p] )[0]
            pxlist = timg[  p[w]   ] -1
            mean_intensity[n] = np.bincount( qind[pxlist], weights = v[w], minlength = len(index)+1 )[1:]
            n +=1
    else:
        ring_masks = [   np.array(labeled_array==i, dtype = np.int64)  for i in np.unique( labeled_array )[1:] ]
        inputs = range( len(ring_masks) )
        go_through_FD(FD)
        pool =  Pool(processes= len(inputs) )
        print( 'Starting assign the tasks...')
        results = {}
        for i in tqdm ( inputs ):
            results[i] = apply_async( pool,  _get_mean_intensity_one_q, ( FD, sampling,   ring_masks[i] ) )
        pool.close()
        print( 'Starting running the tasks...')
        res =   [ results[k].get() for k in   tqdm( list(sorted(results.keys())) )   ]
        #return res
        for i in inputs:
            mean_intensity[:,i] = res[i]
        print( 'ROI mean_intensit calculation is DONE!')
        del results
        del res

    mean_intensity /= norm
    return mean_intensity, index

def _get_mean_intensity_one_q( FD, sampling,   labels ):
    mi = np.zeros(     int( ( FD.end - FD.beg)/sampling )   )
    n=0
    qind, pixelist = roi.extract_label_indices(  labels  )
    # iterate over the images to compute multi-tau correlation
    fra_pix = np.zeros_like( pixelist, dtype=np.float64)
    timg = np.zeros(    FD.md['ncols'] * FD.md['nrows']   , dtype=np.int32   )
    timg[pixelist] =   np.arange( 1, len(pixelist) + 1  )
    for  i in range( FD.beg, FD.end, sampling  ):
        (p,v) = FD.rdrawframe(i)
        w = np.where( timg[p] )[0]
        pxlist = timg[  p[w]   ] -1
        mi[n] = np.bincount( qind[pxlist], weights = v[w], minlength = 2 )[1:]
        n +=1
    return mi

def get_each_frame_intensityc( FD, sampling = 1,
                             bad_pixel_threshold=1e10, bad_pixel_low_threshold=0,
                              hot_pixel_threshold=2**30,
                             plot_ = False, bad_frame_list=None, save=False, *argv,**kwargs):
    '''Get the total intensity of each frame by sampling every N frames
       Also get bad_frame_list by check whether above  bad_pixel_threshold

       Usuage:
       imgsum, bad_frame_list = get_each_frame_intensity(good_series ,sampling = 1000,
                                bad_pixel_threshold=1e10,  plot_ = True)
    '''

    #print ( argv, kwargs )
    #mask &= img < hot_pixel_threshold
    imgsum  =  np.zeros(  int(  (FD.end - FD.beg )/ sampling )  )
    n=0
    for  i in tqdm(range( FD.beg, FD.end, sampling  ), desc= 'Get each frame intensity' ):
        (p,v) = FD.rdrawframe(i)
        if len(p)>0:
            imgsum[n] = np.sum(  v )
        n += 1

    if plot_:
        uid = 'uid'
        if 'uid' in kwargs.keys():
            uid = kwargs['uid']
        fig, ax = plt.subplots()
        ax.plot(   imgsum,'bo')
        ax.set_title('uid= %s--imgsum'%uid)
        ax.set_xlabel( 'Frame_bin_%s'%sampling )
        ax.set_ylabel( 'Total_Intensity' )

        if save:
            #dt =datetime.now()
            #CurTime = '%s%02d%02d-%02d%02d-' % (dt.year, dt.month, dt.day,dt.hour,dt.minute)
            path = kwargs['path']
            if 'uid' in kwargs:
                uid = kwargs['uid']
            else:
                uid = 'uid'
            #fp = path + "uid= %s--Waterfall-"%uid + CurTime + '.png'
            fp = path + "uid=%s--imgsum-"%uid  + '.png'
            fig.savefig( fp, dpi=fig.dpi)

        plt.show()

    bad_frame_list_ = np.where(   ( np.array(imgsum) > bad_pixel_threshold ) | ( np.array(imgsum) <= bad_pixel_low_threshold)  )[0] +  FD.beg

    if  bad_frame_list is not None:
        bad_frame_list = np.unique(   np.concatenate([bad_frame_list, bad_frame_list_]) )
    else:
        bad_frame_list = bad_frame_list_

    if len(bad_frame_list):
        print ('Bad frame list length is: %s' %len(bad_frame_list))
    else:
        print ('No bad frames are involved.')
    return imgsum,bad_frame_list


import os
import sys
import procedural_city_generation
donemessage = "\n"+(150*"-")+"\n\t\t\t  Done, waiting for command\n"+(150*"-")+"\n"
path = os.path.dirname(procedural_city_generation.__file__)
sys.path.append(path)
if not os.path.exists(path+"/temp/"):
    os.system("mkdir "+path+"/temp")
if not os.path.exists(path+"/outputs/"):
    os.system("mkdir "+path+"/outputs")

def setup_matplotlib():
    """
    This function is used to set the matplotlib backend correctly.

    Parameters
    ----------

    Returns
    --------
    None

    :return:
    """
    if sys.version[0] == "3":
        import matplotlib
        try:
            matplotlib.use("Qt4Agg")
        except:
            print("PyQt4 is not installed - outputs will only be saved as images and not be visible at runtime")
            print("However, it is strongly recommended that you install PyQt4 in order to use the GUI")
            matplotlib.use("agg")

from procedural_city_generation.roadmap import main as roadmap_main
from procedural_city_generation.polygons import main as polygons_main
from procedural_city_generation.building_generation import main as building_generation_main
from procedural_city_generation.additional_stuff.Singleton import Singleton

def setRoadmapGUI(gui):
    roadmap_main.gui = gui
    Singleton("roadmap").kill()

def setPolygonsGUI(gui):
    polygons_main.gui = gui
    Singleton("polygons").kill()

def setBuilding_generationGUI(gui):
    building_generation_main.gui = gui
    Singleton("building_generation").kill()

def roadmap():
    roadmap_main.main()
    Singleton("roadmap").kill()
    print(donemessage)

def polygons():
    polygons_main.main(None)
    Singleton("polygons").kill()
    print(donemessage)

def building_generation():
    building_generation_main.main()
    Singleton("building_generation").kill()
    print(donemessage)

def visualization():
    os.system("blender --python "+path+"/visualization/blenderize.py")
    from procedural_city_generation.additional_stuff.Singleton import Singleton
    Singleton("visualization").kill()

def main(args):
    """

    Welcome to procedural_city_generation, a module for procedurally generating a 3D model of a city in Blender with python.

    A call to this module from the command line should follow this format::

        python UI.py <submodule-name> <options>

    <submodule-name> is either "roadmap", "polygons", "building_generation, "visualization".
    <options> is either "run" or "configure"

    If you want to configure a paremeter, go with

        python UI.py <submodule-name> --configure <parameter-name> <new value>

    """
    if len(args) == 1:
        print(main.__doc__)
        return 0
    if "configure" in args[2]:
        config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                   "procedural_city_generation/inputs/{0}.conf".format(args[1]))
        if len(args) == 3:
            os.system("nano {0}".format(config_file))
            sys.exit(0)

        elif args[3] and args[4]:
            import json
            with open(config_file, 'r') as f:
                wb = json.loads(f.read())
            i = 0
            while True:
                try:
                    old = wb[args[3+i]]['value']
                    wb[args[3+i]]['value'] = eval(args[4+i])
                    print("{0} was changed from {1} to {2}".format(args[3+i], old, args[4+i]))
                    i += 2
                    if len(args)-1 < i+4:
                        break

                except:
                    print(i, len(args))
                    print("Either {0} is not a configurable parameter for {1}".format(args[3+i], args[1]))
                    return 0

            with open(config_file, 'w') as f:
                f.write(json.dumps(wb, indent=2))

            return 0

    elif "run" in args[2]:
        setup_matplotlib()
        eval(args[1])()

if __name__ == '__main__':
    main(sys.argv)


import pandas as pd
import numpy as np
import scipy.stats as st
DAYS_IN_YEAR=256.0
ROOT_DAYS_IN_YEAR=DAYS_IN_YEAR**.5

useroot=""

def cap_forecast(xrow, capmin,capmax):
    """
    Cap forecasts.

    """

    ## Assumes we have a single column
    x=xrow[0]

    if x<capmin:
        return capmin
    elif x>capmax:
        return capmax

    return x

def cap_series(xseries, capmin=-20.0,capmax=20.0):
    """
    Apply capping to each element of a time series
    For a long only investor, replace -20.0 with 0.0

    """
    return xseries.apply(cap_forecast, axis=1, args=(capmin, capmax))

def get_list_code():
    ans=pd.read_csv("%sconfig.csv" % useroot)
    return list(ans.Instrument)

def get_point_sizes():
    ans=pd.read_csv("%sconfig.csv" % useroot)
    psizes=dict([(x[1].Instrument, float(x[1].Pointsize)) for x in ans.iterrows()])
    return psizes

def pd_readcsv(filename):
    """
    Reads the pandas dataframe from a filename, given the index is correctly labelled
    """

    ans=pd.read_csv(filename)

    ans.index=pd.to_datetime(ans['DATETIME'])
    del ans['DATETIME']
    ans.index.name=None
    return ans

def find_datediff(data_row):
    """
    data differential for a single row
    """
    if np.isnan(data_row.NEAR_MONTH) or np.isnan(data_row.TRADE_MONTH):
        return np.nan
    nearest_dt=pd.to_datetime(str(int(data_row.NEAR_MONTH)), format="%Y%m")
    trade_dt=pd.to_datetime(str(int(data_row.TRADE_MONTH)), format="%Y%m")

    distance = trade_dt - nearest_dt
    distance_years=distance.days/365.25

    ## if nearder contract is cheaper; price will fall
    price_diff=data_row.NEARER - data_row.TRADED

    return price_diff/distance_years

def ewmac_forecast_scalar(Lfast, Lslow):
    """
    Function to return the forecast scalar (table 49 of the book)

    Only defined for certain values
    """

    fsdict=dict(l2_8=10.6, l4_16=7.5, l8_32=5.3, l16_64=3.75, l32_128=2.65, l64_256=1.87)

    lkey="l%d_%d" % (Lfast, Lslow)

    if lkey in fsdict:
        return fsdict[lkey]
    else:
        print ("Warning: No scalar defined for Lfast=%d, Lslow=%d, using default of 1.0" % (Lfast, Lslow))
        return 1.0

def get_price_for_instrument(code):

    filename="%sdata/%s_price.csv" %  (useroot, code)
    price=pd_readcsv(filename)
    return price

def get_carry_data(code):

    filename="%sdata/%s_carrydata.csv" % (useroot, code)
    data=pd_readcsv(filename)

    return data

def uniquets(df3):
    """
    Makes x unique
    """
    df3=df3.groupby(level=0).first()
    return df3

def daily_resample(b, a):
    """
    Returns b dataframe resampled to a dataframe index

    """

    master_index=a.index
    a_daily=a.resample('1D') ## Only want index, fill method is irrelevant
    b=uniquets(b)
    b_daily=b.reindex(a_daily.index, method="ffill", limit=1)
    new_b=b_daily.reindex(master_index, method="ffill", limit=1)

    return new_b

def calculate_pandl(position_ts, price_ts, pointsize=1.0):
    rs_positions_ts=daily_resample(position_ts, price_ts).ffill()
    rets=price_ts - price_ts.shift(1)
    local_rets=rs_positions_ts.shift(1)*rets*pointsize

    return local_rets

def annualised_rets(total_rets):
    mean_rets=total_rets.mean(skipna=True)
    annualised_rets=mean_rets*DAYS_IN_YEAR
    return annualised_rets

def annualised_vol(total_rets):
    actual_total_daily_vol=total_rets.std(skipna=True)
    actual_total_annual_vol=actual_total_daily_vol*ROOT_DAYS_IN_YEAR
    return actual_total_annual_vol

def sharpe(total_rets):

    sharpe=annualised_rets(total_rets)/annualised_vol(total_rets)

    return sharpe

def stack_ts(tslist, start_date=pd.datetime(1970,1,1)):

    """
    Take a list of time series, and stack them, generating a new time series
    """

    tslist_values=[list(x.iloc[:,0].values) for x in tslist]
    stack_values=sum(tslist_values, [])
    stack_values=[x for x in stack_values if not np.isinf(x)]

    stacked=arbitrary_timeindex(stack_values, start_date)

    return stacked

def slices_for_ts(data, freq="12M"):
    """
    Return date indices for slicing up a data frame
    """
    yridx=list(pd.date_range(start=data.index[0], end=data.index[-1], freq=freq))
    yridx_stub=list(pd.date_range(start=yridx[-1], periods=2, freq=freq))[-1]
    yridx=yridx+[yridx_stub]

    return yridx

def break_up_ts(data, freq="12M"):
    """
    Take a data frame and break it into chunks
    returns a list of data frames
    """
    yridx=slices_for_ts(data, freq)

    brokenup=[]
    for idx in range(len(yridx))[1:]:
        brokenup.append(data[yridx[idx-1]:yridx[idx]])

    return brokenup

def drawdown(x):
    ### Returns a ts of drawdowns for a time series x

    ## rolling max with infinite window
    maxx=pd.rolling_max(x, 99999999, min_periods=1)
    return (x - maxx)/maxx

class account_curve(pd.core.series.Series):
    """
    Inherits from pandas time series to give useful information

    Could be in % or GBP terms

    Downsamples to daily before doing anything else

    Can

    """

    def new_freq(self, freq):
        ## Set up a new frequency.
        ## Note this will break certain things (eg Sharpe) so be careful
        if freq=="Daily":
            ## we assume we're daily so do nothing
            return self
        if freq=="Weekly":
            return self.cumsum().ffill().resample("W").diff()
        if freq=="Monthly":
            return self.cumsum().ffill().resample("M").diff()

    def sharpe(self):
        ## assumes daily returns
        return ROOT_DAYS_IN_YEAR*self.mean()/self.std()

    def annstd(self):
        return ROOT_DAYS_IN_YEAR*self.std()

    def losses(self):
        x=self.values
        return [z for z in x if z<0]

    def gains(self):
        x=self.values
        return [z for z in x if z>0]

    def avg_loss(self):
        return np.mean(self.losses())

    def avg_gain(self):
        return np.mean(self.gains())

    def drawdown(self):
        ## in case need numerous stats
        if "drawdownacc" not in dir(self):
            setattr(self, "drawdownacc", drawdown(cum_perc(self)))
        return self.drawdownacc

    def avg_drawdown(self):
        return self.perc_drawdown(50.0)

    def perc_drawdown(self, q):
        dd=self.drawdown()
        return np.percentile(dd, q)

    def worst_drawdown(self):
        dd=self.drawdown()
        return np.nanmin(dd.values)

    def time_in_drawdown(self):
        dd=self.drawdown()
        dd=[z for z in dd if not np.isnan(z)]
        in_dd=float(len([z for z in dd if z<0]))
        return in_dd/float(len(dd))

    def monthly_returns(self):
        return self.resample("1M", how="sum")

    def gaintolossratio(self):
        return self.avg_gain()/-self.avg_loss()

    def profitfactor(self):
        return sum(self.gains())/-sum(self.losses())

    def hitrate(self):
        no_gains=float(len(self.gains()))
        no_losses=float(len(self.losses()))
        return no_gains/(no_losses+no_gains)

def cum_perc(pd_timeseries):
    """
    Cumulate percentage returns for a pandas time series
    """

    cum_datalist=[1+x for x in pd_timeseries]
    cum_datalist=pd.TimeSeries(cum_datalist, index=pd_timeseries.index)

    return cum_datalist.cumprod()

def arbitrary_timeindex(Nperiods, index_start=pd.datetime(2000,1,1)):
    """
    For nice plotting, convert a list of prices or returns into an arbitrary pandas time series
    """

    ans=pd.bdate_range(start=index_start, periods=Nperiods)

    return ans

def arbitrary_timeseries(datalist, index_start=pd.datetime(2000,1,1)):
    """
    For nice plotting, convert a list of prices or returns into an arbitrary pandas time series
    """

    ans=pd.TimeSeries(datalist, index=arbitrary_timeindex(len(datalist), index_start))

    return ans

def remove_nans_from_list(xlist):
    return [x for x in xlist if not np.isnan(x)]

def autocorr(x, t=1):
    return np.corrcoef(np.array([x[0:len(x)-t], x[t:len(x)]]))[0,1]


import numpy as np

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import TempMemmap

from sklearn.decomposition import DictionaryLearning
from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.decomposition import SparseCoder
from sklearn.decomposition import dict_learning_online
from sklearn.decomposition import sparse_encode

rng_global = np.random.RandomState(0)
n_samples, n_features = 10, 8
X = rng_global.randn(n_samples, n_features)

def test_dict_learning_shapes():
    n_components = 5
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_overcomplete():
    n_components = 12
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_reconstruction():
    n_components = 12
    dico = DictionaryLearning(n_components, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

    # used to test lars here too, but there's no guarantee the number of
    # nonzero atoms is right.

def test_dict_learning_reconstruction_parallel():
    # regression test that parallel reconstruction works with n_jobs=-1
    n_components = 12
    dico = DictionaryLearning(n_components, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0, n_jobs=-1)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

def test_dict_learning_lassocd_readonly_data():
    n_components = 12
    with TempMemmap(X) as X_read_only:
        dico = DictionaryLearning(n_components, transform_algorithm='lasso_cd',
                                  transform_alpha=0.001, random_state=0, n_jobs=-1)
        code = dico.fit(X_read_only).transform(X_read_only)
        assert_array_almost_equal(np.dot(code, dico.components_), X_read_only, decimal=2)

def test_dict_learning_nonzero_coefs():
    n_components = 4
    dico = DictionaryLearning(n_components, transform_algorithm='lars',
                              transform_n_nonzero_coefs=3, random_state=0)
    code = dico.fit(X).transform(X[1])
    assert_true(len(np.flatnonzero(code)) == 3)

    dico.set_params(transform_algorithm='omp')
    code = dico.transform(X[1])
    assert_equal(len(np.flatnonzero(code)), 3)

def test_dict_learning_unknown_fit_algorithm():
    n_components = 5
    dico = DictionaryLearning(n_components, fit_algorithm='<unknown>')
    assert_raises(ValueError, dico.fit, X)

def test_dict_learning_split():
    n_components = 5
    dico = DictionaryLearning(n_components, transform_algorithm='threshold',
                              random_state=0)
    code = dico.fit(X).transform(X)
    dico.split_sign = True
    split_code = dico.transform(X)

    assert_array_equal(split_code[:, :n_components] -
                       split_code[:, n_components:], code)

def test_dict_learning_online_shapes():
    rng = np.random.RandomState(0)
    n_components = 8
    code, dictionary = dict_learning_online(X, n_components=n_components,
                                            alpha=1, random_state=rng)
    assert_equal(code.shape, (n_samples, n_components))
    assert_equal(dictionary.shape, (n_components, n_features))
    assert_equal(np.dot(code, dictionary).shape, X.shape)

def test_dict_learning_online_verbosity():
    n_components = 5
    # test verbosity
    from sklearn.externals.six.moves import cStringIO as StringIO
    import sys

    old_stdout = sys.stdout
    try:
        sys.stdout = StringIO()
        dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=1,
                                           random_state=0)
        dico.fit(X)
        dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=2,
                                           random_state=0)
        dico.fit(X)
        dict_learning_online(X, n_components=n_components, alpha=1, verbose=1,
                             random_state=0)
        dict_learning_online(X, n_components=n_components, alpha=1, verbose=2,
                             random_state=0)
    finally:
        sys.stdout = old_stdout

    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_estimator_shapes():
    n_components = 5
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, random_state=0)
    dico.fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_overcomplete():
    n_components = 12
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20,
                                       random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_initialization():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)
    dico = MiniBatchDictionaryLearning(n_components, n_iter=0,
                                       dict_init=V, random_state=0).fit(X)
    assert_array_equal(dico.components_, V)

def test_dict_learning_online_partial_fit():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    dict1 = MiniBatchDictionaryLearning(n_components, n_iter=10 * len(X),
                                        batch_size=1,
                                        alpha=1, shuffle=False, dict_init=V,
                                        random_state=0).fit(X)
    dict2 = MiniBatchDictionaryLearning(n_components, alpha=1,
                                        n_iter=1, dict_init=V,
                                        random_state=0)
    for i in range(10):
        for sample in X:
            dict2.partial_fit(sample)

    assert_true(not np.all(sparse_encode(X, dict1.components_, alpha=1) ==
                           0))
    assert_array_almost_equal(dict1.components_, dict2.components_,
                              decimal=2)

def test_sparse_encode_shapes():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'):
        code = sparse_encode(X, V, algorithm=algo)
        assert_equal(code.shape, (n_samples, n_components))

def test_sparse_encode_error():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = sparse_encode(X, V, alpha=0.001)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)

def test_sparse_encode_error_default_sparsity():
    rng = np.random.RandomState(0)
    X = rng.randn(100, 64)
    D = rng.randn(2, 64)
    code = ignore_warnings(sparse_encode)(X, D, algorithm='omp',
                                          n_nonzero_coefs=None)
    assert_equal(code.shape, (100, 2))

def test_unknown_method():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    assert_raises(ValueError, sparse_encode, X, V, algorithm="<unknown>")

def test_sparse_coder_estimator():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = SparseCoder(dictionary=V, transform_algorithm='lasso_lars',
                       transform_alpha=0.001).transform(X)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)


"""
========================================
Lasso and Elastic Net for Sparse Signals
========================================

Estimates Lasso and Elastic-Net regression models on a manually generated
sparse signal corrupted with an additive noise. Estimated coefficients are
compared with the ground-truth.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.metrics import r2_score

###############################################################################
# generate some sparse data to play with
np.random.seed(42)

n_samples, n_features = 50, 200
X = np.random.randn(n_samples, n_features)
coef = 3 * np.random.randn(n_features)
inds = np.arange(n_features)
np.random.shuffle(inds)
coef[inds[10:]] = 0  # sparsify coef
y = np.dot(X, coef)

# add noise
y += 0.01 * np.random.normal((n_samples,))

# Split data in train set and test set
n_samples = X.shape[0]
X_train, y_train = X[:n_samples / 2], y[:n_samples / 2]
X_test, y_test = X[n_samples / 2:], y[n_samples / 2:]

###############################################################################
# Lasso
from sklearn.linear_model import Lasso

alpha = 0.1
lasso = Lasso(alpha=alpha)

y_pred_lasso = lasso.fit(X_train, y_train).predict(X_test)
r2_score_lasso = r2_score(y_test, y_pred_lasso)
print(lasso)
print("r^2 on test data : %f" % r2_score_lasso)

###############################################################################
# ElasticNet
from sklearn.linear_model import ElasticNet

enet = ElasticNet(alpha=alpha, l1_ratio=0.7)

y_pred_enet = enet.fit(X_train, y_train).predict(X_test)
r2_score_enet = r2_score(y_test, y_pred_enet)
print(enet)
print("r^2 on test data : %f" % r2_score_enet)

plt.plot(enet.coef_, color='lightgreen', linewidth=2,
         label='Elastic net coefficients')
plt.plot(lasso.coef_, color='gold', linewidth=2,
         label='Lasso coefficients')
plt.plot(coef, '--', color='navy', label='original coefficients')
plt.legend(loc='best')
plt.title("Lasso R^2: %f, Elastic Net R^2: %f"
          % (r2_score_lasso, r2_score_enet))
plt.show()


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 06 20:00:30 2016

@author: Jiansen
"""
import requests
import threading
import copy
import logging
import os
#import urllib3
import json
from scipy import stats
#from decimal import Decimal, getcontext, ROUND_HALF_DOWN

#from event00 import TickEvent,TickEvent2
#import time
import oandapy
import httplib
import pandas as pd
import math
import numpy as np
import pywt
import time
from settings import STREAM_DOMAIN, API_DOMAIN, ACCESS_TOKEN, ACCOUNT_ID

from trading_events import Event,TickEvent2,LiqEvent,OrderEvent,CloseEvent
from trading_global_functions import *
from trading_log import log_dict

import Queue

#for writing data
import datetime
from bson.objectid import ObjectId
import pymongo as pm
from pymongo import MongoClient
import statsmodels.tsa.stattools as ts
#requests.adapters.DEFAULT_RETRIES = 5
from warningOps import warning
from seriesADF import getADF

corpid=  ''
secret=''
warn = warning(corpid,secret)

#------the only line we need to change is about the instruments----#
pairs = "EUR_USD"
#-----------------------------------------------------------------------#

client = MongoClient('localhost',27017)
db = client.test_database

#---------------Initialize the parameters and database connections-------#
if pairs == "EUR_USD":
    try:
        from param_EUR_USD import MA_dict, threshold_dict,sltp_dict
    except ImportError:
        raise ValueError("cannot find parameters for {0}!".format(pairs))
    collection = db.tick_test
    index_collect = db.index_EUR_USD
elif pairs == "USD_CNH":
    try:
        from param_USD_CNH import MA_dict, threshold_dict,sltp_dict
    except ImportError:
        raise ValueError("cannot find parameters for {0}!".format(pairs))
    collection = db.tick_USD_CNH
    index_collect = db.index_USD_CNH

elif pairs == "AUD_USD":
    try:
        from param_AUD_USD import MA_dict, threshold_dict,sltp_dict
    except ImportError:
        raise ValueError("cannot find parameters for {0}!".format(pairs))
    collection = db.tick_AUD_USD
    index_collect = db.index_AUD_USD

else:
    raise ValueError('Invalid <pairs>, CANNOT FIND THE INSTRUMENTS!')
#-----------------------------------------------------------------------#

#--------------------------Liquidity Index------------------------------#
#-----------------------------------------------------------------------#
class LiqForex(object):
    def __init__(
        self, domain, access_token,
        account_id, instruments,ct, gran, dd, events_queue
    ):
        self.domain = domain
        self.access_token = access_token
        self.account_id = account_id
        self.instruments = instruments
        self.ct = ct
        self.gran=gran
        self.dd= dd
        self.events_queue = events_queue

    def getLiq(self):
        try:
            requests.packages.urllib3.disable_warnings()
            s = requests.Session()
            #s.keep_alive = False
            url = "https://" + self.domain + "/v1/candles"
            headers = {'Authorization' : 'Bearer ' + self.access_token}
            params = {'instrument':self.instruments, 'accountId' : self.account_id,
                      'count':self.ct,'candleFormat':'midpoint','granularity':self.gran}
            req = requests.Request('GET', url, headers=headers, params=params)
            pre = req.prepare()
            logging.info( pre)
            resp = s.send(pre, stream=False, verify=False)
            try:
                msg=json.loads(resp.text)
            except Exception as e:
                logging.warning( "Caught exception when converting message into json\n" + str(e))
                return
            if msg.has_key("candles"):
                time0=msg.get("candles")[-1]["time"]
                lis = ohlcv_lis(msg.get("candles"))
                liqS = pd.Series()
                for i in range(0, len(lis)- (self.dd+1) ,1):
                    s2 = liq15min(lis[i:i+self.dd])
                    liqS = np.append(liqS,s2)
                liq=liqS[-1]
                logging.info( "liq=".format(liq))
                tev = LiqEvent(self.instruments,time0,liq)
                self.events_queue.put(tev,False)
                post_metric = get_indicator(self.instruments,None,None,self.gran,liq,None,None)
                index_collect.insert_one(post_metric)
        except Exception as e:
            s.close()
            content0 = "Caught exception when connecting to history\n" + str(e)
            logging.warning(content0)
            #warn.tradingWarning(content0)
    def activeLiq(self,period):
        while True:
            self.getLiq()
            time.sleep(period)
#--------------------------------------------------------------------#
class StreamingForexPrices(object):
    def __init__(
        self, domain, access_token,
        account_id, instruments,ct, gran, dd, events_queue
    ):
        self.domain = domain
        self.access_token = access_token
        self.account_id = account_id
        self.instruments = instruments
        self.ct = ct
        self.gran=gran
        self.dd= dd
        self.events_queue = events_queue

    def connect_to_stream(self):
        try:
            requests.packages.urllib3.disable_warnings()
            s = requests.Session()  # socket
            url = "https://" + self.domain + "/v1/prices"
            headers = {'Authorization' : 'Bearer ' + self.access_token}
            params = {'instruments' : self.instruments, 'accountId' : self.account_id}
            time.sleep(0.8) # sleep some seconds
            req = requests.Request('GET', url, headers=headers, params=params)
            pre = req.prepare()
            resp = s.send(pre, stream=True, verify=False)
            return resp
        except Exception as e:
            #global s
            s.close()
            content0 = "Caught exception when connecting to stream\n" + str(e)
            logging.warning(content0)
            #warn.tradingWarning(content0)

    def stream_to_queue_old(self,collection):
        response = self.connect_to_stream()
        if response.status_code != 200:
            return
        try:
            for line in response.iter_lines(1):
                if line:
                    try:
                        msg = json.loads(line)
                    except Exception as e:
                        content0 = "Caught exception when converting message into json\n" + str(e)
                        logging.warning(content0)
                        return
                    if msg.has_key("instrument") or msg.has_key("tick"):
                        logging.info(msg)
                        instrument = msg["tick"]["instrument"]
                        time0 = msg["tick"]["time"]
                        bid = msg["tick"]["bid"]
                        ask = msg["tick"]["ask"]
                        tev = TickEvent2(instrument, time0, bid, ask)
                        self.events_queue.put(tev,False)
                        post= getDoc(msg)
                        collection.insert_one(post)
        except Exception as e:
            logging.warning('Caught ChunkedEncodingError in  stream_to_queue_old()!'+str(time.ctime()))
            return

#--------------
#------
 # new strategy
class LiqMAStrategy(object):
    """
    """
    def __init__(
        self, access_token, account_id, pairs, units, events, stopLoss1, takeProfit1,stopLoss2, takeProfit2,
        short_window1, long_window1,short_window2, long_window2, idxU, lam, thres1, thres2,thres3, thres4, adf_thres
    ):
        self.access_token = access_token
        self.account_id = account_id

        self.pairs = pairs
        self.units = units
        self.stopLoss1 = stopLoss1
        self.takeProfit1 = takeProfit1
        self.stopLoss2 = stopLoss2
        self.takeProfit2 = takeProfit2
        self.pairs_dict = self.create_pairs_dict()
        self.events = events
        self.short_window1 = short_window1
        self.long_window1 = long_window1
        self.short_window2 = short_window2
        self.long_window2 = long_window2
        self.idxU = idxU
        self.lam = lam
        self.priceLis1 = pd.Series()    #for trends
        self.priceLis2 = pd.Series()    #for reversion
        self.thres1 = thres1
        self.thres2 = thres2
        self.thres3 = thres3
        self.thres4 = thres4
        self.adf_thres = adf_thres
        #---intermediates---#
        self.SL_TP = {"trends":[self.stopLoss1,self.takeProfit1],
                    "reversion":[self.stopLoss2,self.takeProfit2]}
        self.s_l_window = {"trends":[self.short_window1,self.long_window1],
                    "reversion":[self.short_window2,self.long_window2]}
        self.thres_tre_rev = {"trends":[self.thres1, self.thres2],
                            "reversion":[self.thres3,self.thres4]}

    def create_pairs_dict(self):
        attr_dict = {
            "ticks": 0,
            "tick0": 0,
            "priceLS":0.0,
            "invested": False,
            "short_sma": None,
            "long_sma": None,
            "longShort": None,
            "short_slope":None,
            "long_slope":None,  # False denotes sell, while True denotes buy
            "check": False,
            "orlis":[0,0,0,0],
            "stra": 0,
            "fixed": False
        }
        #pairs_dict = {}
        pairs_dict = copy.deepcopy(attr_dict)
        return pairs_dict

    def check_order(self,check):
        if check== True:
            oanda0 = oandapy.API(environment="practice", access_token=self.access_token)
            try:
                responseTrades = oanda0.get_trades(self.account_id,instrument=self.pairs)
            except Exception as e:
                logging.warning('Caught exception in get_trades() of check_order()!\n'+str(time.ctime()))
                return
            if responseTrades.get("trades")==[]:
                pd = self.pairs_dict
                pd["orlis"].pop(0)
                logging.info(" orlis: "+str(pd["orlis"]))
                pd["orlis"].append(0)
                logging.info(" orlis: "+str(pd["orlis"]))
                if pd["orlis"][0:4]==[1,1,0,0]:
                    logging.warning( "Stop Loss Order Executed!")
                    #warn.tradingWarning(" Stop Loss Order Executed!")
                    pd["invested"]= False
                    pd["fixed"] = False  #position closed, the stra type is free
                    pd["check"] = False
                else:
                    pass
            else:
                pd = self.pairs_dict
                #pd["orlis"][0] = copy.copy(pd["orlis"][1])
                pd["orlis"].pop(0)
                pd["orlis"].append(1)
                logging.info("not empty- orlis: "+str(pd["orlis"]))
                pd["invested"]= True
                pd["fixed"] = True  #position closed, the stra type is free
                pd["check"] = True
        else:
            pass

    def compute_slope(self,price_lis,window_length,k):
        '''[summary]
        compute the slope ratio for a short time series
        Arguments:
            price_lis {np.ndarray} -- the filtered time series to compute the slope ratio
            for both SMA and LMA
            default: newPriceLis
            window_length {[type]} -- a parameter for the SMA
            k: an parameter for performing average, default->0.5
            default: self.short_window2
        Returns:
            [float] -- [the slope ratio]
        '''
        amp = lambda lis: (lis-lis[0])*10000.0
        pShort =   amp(price_lis[-window_length:])
        pLong =  amp(price_lis)
        #compute the slope ratio
        aveSlope = k*getSlope(pShort)+ (1-k)*getSlope(pLong)
        return aveSlope

    def set_invested_check_fixed(self,pair_dict,invested_bool,check_bool,fixed_bool):
        pair_dict["invested"] = invested_bool
        pair_dict["check"] = check_bool
        pair_dict["fixed"] = fixed_bool
        time.sleep(0.0)

    def get_sl_tp(self,TreRev):
        return self.SL_TP[TreRev]

    def insert_metric(self,collection,pair_dict):
        '''
        default collection: index_USD_CNH
        '''
        short_window,long_window = self.s_l_window[pair_dict["stra"]]
        post_metric = get_indicator(self.pairs,short_window,long_window,
                                    None,None,pair_dict["short_slope"],pair_dict["long_slope"])
        collection.insert_one(post_metric)
#----------------#
    def buy_send_order(self,pd,side,price0,price1,TreRev):
        logging.info("price02={0}".format(price0))
        self.set_invested_check_fixed(pd,True,True,True)
        fixSL, fixeTP = self.get_sl_tp(TreRev)
        sl_b, tp_b= round(price0 - fixSL,5),round(price1 + fixeTP,5)
        order = OrderEvent(self.pairs, self.units, "market", side, sl_b, tp_b,"Trends")
        self.events.put(order)
        pd["longShort"] = True
        pd["tick0"]= pd["ticks"]
        pd["priceLS"]= price0

    def sell_send_order(self,pd,side,price0,price1,TreRev):
        logging.info("price01={0}".format(price1))
        self.set_invested_check_fixed(pd,True,True,True)
        fixSL, fixeTP = self.get_sl_tp(TreRev)
        sl_s,tp_s = round(price1 + fixSL,5),round(price0 - fixeTP,5)
        order = OrderEvent(self.pairs, self.units, "market", side, sl_s, tp_s,"Trends")
        self.events.put(order)
        pd["longShort"] = False
        pd["tick0"]= pd["ticks"]
        pd["priceLS"]= price1

    def logging_invested(self,priceLis,pd,sign):
        TreRev = pd["stra"]
        logging.info(TreRev+" position!")
        #??? TODO  23:38 Oct 5, 2017
        short_window = self.s_l_window[TreRev][0]
        newPriceLis = get_new_price_lis(priceLis, pd, short_window)
        basePrice=pd["priceLS"]+sign*self.lam*np.std(priceLis)*np.sqrt(pd["ticks"]-pd["tick0"])
        logging.info( "basePrice="+str(basePrice))
        logging.info( "short_sma"+str(pd["short_sma"]))
        logging.info( "long_sma"+str(pd["long_sma"]))
        aveSlope = self.compute_slope(newPriceLis,short_window, 0.5)
        logging.info( "aveSlope="+str(aveSlope))
        return aveSlope

    def put_close_order(self,pairs,num):
        '''
        pairs,num = self.pairs,0
        '''
        order_closed = CloseEvent(pairs,num)
        self.events.put(order_closed)
#--------------------------------------#
    def open_trends_buy(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][0]
        return (pd["short_sma"] > pd["long_sma"] and aveSlope > thres)

    def open_trends_sell(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][0]
        return (pd["short_sma"] < pd["long_sma"] and aveSlope < -thres)

    def open_reversion_buy(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][0]
        return (pd["short_sma"] < pd["long_sma"] and aveSlope< -thres)

    def open_reversion_sell(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][0]
        return (pd["short_sma"] > pd["long_sma"] and aveSlope> thres)
#-----------------------------------------------#
    def close_trends_buy(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][1]
        return (pd["longShort"] and aveSlope < thres)

    def close_trends_sell(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][1]
        return (not pd["longShort"]  and aveSlope > -thres)

    def close_reversion_buy(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][1]
        return (pd["short_sma"] > pd["long_sma"]*(1+thres/100.0)  and pd["longShort"])

    def close_reversion_sell(self,pd,aveSlope):
        thres = self.thres_tre_rev[pd["stra"]][1]
        return (pd["short_sma"] < pd["long_sma"]*(1-thres/100.0)  and not pd["longShort"])
#--------------------------------------#
    def calculate_signals(self, event):
        #if True:
        global liqIndex
        global newPriceLis
        if event.type == 'TICK':
            price = (event.bid+event.ask)/2.000
            self.priceLis1 = np.append(self.priceLis1,price)
            self.priceLis2 = np.append(self.priceLis2,price)
            if len(self.priceLis1)>max([self.long_window1,self.long_window2]):
                self.priceLis1=self.priceLis1[-self.long_window1:]
                self.priceLis2=self.priceLis2[-self.long_window2:]
            else:
                pass
            #liqIndex= event.liq
            logging.info("liqIndex= "+str(liqIndex)+"\n")
            logging.info("price= "+str(price))
            pd = self.pairs_dict
            logging.info("check"+str(pd["check"]))
            self.check_order(pd["check"])   #check whether the SLTP order is triggered..
            # Only start the strategy when we have created an accurate short window
            logging.info("INVESTED= "+str(pd["invested"]))
            if not pd["invested"]:
                #global price0
                if pd["ticks"]>max([self.long_window1, self.long_window2])+1 and liqIndex > self.idxU:
                    if not pd["fixed"]:
                        critAdf = getADF(collection).priceADF(200,1)
                        if critAdf > self.adf_thres:
                            pd["stra"] = "reversion"
                            newPriceLis = get_new_price_lis(self.priceLis2, pd, self.short_window2)
                            aveSlope = self.compute_slope(newPriceLis,self.short_window2, 0.5)
                            logging.info( "REVERSION+aveSlope="+str(aveSlope))
                            self.insert_metric(index_collect,pd)
                        else:
                            pd["stra"] = "trends"
                            newPriceLis = get_new_price_lis(self.priceLis1, pd, self.short_window1)
                            aveSlope = self.compute_slope(newPriceLis,self.short_window1, 0.5)
                            logging.info("TRENDS+aveSlope="+str(aveSlope))
                            self.insert_metric(index_collect,pd)
                    else:
                        raise ValueError("pd[fixed] should be False!")

                    price0, price1 = event.bid, event.ask
                    if pd["stra"] =="trends":
                        if self.open_trends_buy(pd,aveSlope):
                            side = "buy"
                            self.buy_send_order(pd,side,price0,price1,pd["stra"])
                        elif self.open_trends_sell(pd,aveSlope):
                            side = "sell"
                            self.sell_send_order(pd,side,price0,price1,pd["stra"])
                        else:
                            pd["fixed"] = False
                    elif pd["stra"] =="reversion":
                        if self.open_reversion_sell(pd,aveSlope):
                            side = "sell"
                            self.sell_send_order(pd,side,price0,price1,pd["stra"])
                        elif self.open_reversion_buy(pd,aveSlope):
                            side = "buy"
                            self.buy_send_order(pd,side,price0,price1,pd["stra"])
                        else:
                            pd["fixed"] = False
                    else:
                        pass
                else:
                    pass
            elif pd["invested"]:
                sign= 1 if pd["longShort"] == True else -1

                if pd["stra"] =="trends":
                    aveSlope = self.logging_invested(self.priceLis1,pd,sign)
                    self.insert_metric(index_collect,pd)
                    if  self.close_trends_sell(pd,aveSlope):
                        #side = "sell"
                        self.set_invested_check_fixed(pd,False,False,False)
                        #warn.tradingWarning(" check->False Executed!")
                        self.put_close_order(self.pairs,0)

                    elif self.close_trends_buy(pd,aveSlope):
                        #side = "buy"
                        self.set_invested_check_fixed(pd,False,False,False)
                        #warn.tradingWarning(" check->False Executed!")
                        self.put_close_order(self.pairs,0)

                    else: #not closing positions, just keep the pd["fixed"] as True.
                        pd["fixed"] = True  #should we add pd["invested"]

                elif pd["stra"] =="reversion":
                    aveSlope=self.logging_invested(self.priceLis2,pd,sign)
                    self.insert_metric(index_collect,pd)
                    if self.close_reversion_sell(pd,aveSlope):
                        #side = "sell"
                        self.set_invested_check_fixed(pd,False,False,False)
                        #warn.tradingWarning(" check->False Executed!")
                        self.put_close_order(self.pairs,0)

                    elif self.close_reversion_buy(pd,aveSlope):
                        #side = "buy"
                        self.set_invested_check_fixed(pd,False,False,False)
                        #warn.tradingWarning(" check->False Executed!")
                        self.put_close_order(self.pairs,0)

                    else:
                        pd["fixed"] = True  #should we add pd["invested"]
                else:
                    pass

            pd["ticks"] += 1
            logging.info("current Tick "+str(pd["ticks"])+"\n"+str(time.ctime()))

#--------------------------------------------------------------------#
class Execution(object):
    def __init__(self, domain, access_token, account_id):
        self.domain = domain
        self.access_token = access_token
        self.account_id = account_id
        self.conn = self.obtain_connection()

    def obtain_connection(self):
        return httplib.HTTPSConnection(self.domain)

    def execute_order(self, event):
        oanda0 = oandapy.API(environment="practice", access_token=self.access_token)
        try:
            responseX = oanda0.create_order(self.account_id,
            instrument=event.instrument,
            units= event.units,
            side= event.side,
            type= event.order_type,
            stopLoss =   event.stopLoss,
            takeProfit = event.takeProfit
            )
        except Exception as e:
            content0 = "Caught OnadaError when sending the orders\n" + str(e)
            logging.warning(content0)
            return
        logging.info( "Execute Order ! \n {0}".format(responseX))
        content0 = str(event.stra)+"Execute Order ! "+" "+str(event.side)+" "+ str(event.units)+" units of "+str(event.instrument)
        #warn.tradingWarning(content0)
        logging.info(content0)

    def close_order(self, event):
        oanda0 = oandapy.API(environment="practice", access_token=self.access_token)
        response1= oanda0.get_trades(self.account_id,instrument=event.instrument)
        order_lis= response1["trades"]
        if order_lis !=[]:
            for order in order_lis:  #close all trades
                responseX = oanda0.close_trade(self.account_id,trade_id= order['id'])
                logging.info( "Close Order ! \n {0}".format(responseX))
                content0  = "Close Order !" + "profit: "+str(responseX['profit'])+" CLOSE "+str(responseX['instrument'])
                content0  = content0 + " "+str(responseX['side'])+" at "+ str(responseX['price'])
                #warn.tradingWarning(content0)
        else:
            logging.warning("No trade to be closed! :{0}".format(time.ctime()))

#--------------------------------------------------------------------#
def trade(events, strategy,execution,heartbeat):
    """
    """
    global liqIndex
    while True:
        try:
            event = events.get(False)
        except Queue.Empty:
            pass
        else:
            if event is not None:
                if event.type =='LIQ':
                    liqIndex= event.liq
                    #print "current index ="+str(liqIndex)
                elif event.type == 'TICK':
                    strategy.calculate_signals(event)
                    logging.info( "Tick!")
                elif event.type == 'ORDER':
                    logging.info( "Executing order!")
                    execution.execute_order(event)
                elif event.type == "CLOSE":
                    logging.info( "Close trading!")
                    execution.close_order(event)
        time.sleep(heartbeat)
#--------------------------------------------------------------------#
if __name__ == "__main__":

    logPath,logName = log_dict[pairs]["path"],log_dict[pairs]["name"]
    logging.basicConfig(filename= os.path.join(logPath,logName),
                    format='%(levelname)s:%(message)s',level=logging.DEBUG)
    global liqIndex
    liqIndex=0
    ct = 20
    gran ='M15'
    time_dict = {
    "S5": 5,
    "S10": 10,
    "S15": 15,
    "S30": 30,
    "M1": 60,
    "M2": 120 }
    dd = 11
    lam= 0.1     #0.5 basePrice tuning
    units = 100    #100
    #----------Parameters----------------
    short_window1= MA_dict['short_window1']
    long_window1 = MA_dict['long_window1']
    short_window2= MA_dict['short_window2']
    long_window2 = MA_dict['long_window2']

    idxu = threshold_dict['idxu']
    thres1= threshold_dict['thres1']
    thres2= threshold_dict['thres2']
    thres3 = threshold_dict['thres3']
    thres4= threshold_dict['thres4']
    adf_thres =  threshold_dict['adf_thres']

    sl1 = sltp_dict['sl1']   #10
    tp1 = sltp_dict['tp1']   #10
    sl2 = sltp_dict['sl2']   #10
    tp2 = sltp_dict['tp2']   #10
    #--------------------------------------
    heartbeat= 0.2
    period= 600
    print 'initial'
    print('MA:\n sw1 {0} lw1 {1} sw2 {2} lw2 {3}'.format(short_window1, long_window1, short_window2, long_window2))
    print('parameters:\n thres1 {0} thres2 {1} thres3 {2} thres4 {3}'.format(thres1,thres2,thres3,thres4))
    print('sltp_parameters:\n {0} {1} {2} {3}'.format(sl1,tp1,sl2,tp2))
    events = Queue.Queue()
    # initial the threads
    prices = StreamingForexPrices(STREAM_DOMAIN, ACCESS_TOKEN, ACCOUNT_ID, pairs, ct, gran, dd, events)
    liquidity = LiqForex(API_DOMAIN, ACCESS_TOKEN, ACCOUNT_ID, pairs, ct, gran, dd, events)
    execution = Execution(API_DOMAIN, ACCESS_TOKEN, ACCOUNT_ID)
    #strategy = MovingAverageCrossStrategy(pairs, units, events, sl, tp, short_window,long_window)
    strategy = LiqMAStrategy(ACCESS_TOKEN, ACCOUNT_ID, pairs, units, events, sl1, tp1, sl2, tp2, short_window1,long_window1,
        short_window2,long_window2,idxu,lam,thres1,thres2,thres3,thres4,adf_thres)
    # construct the thread
    price_thread = threading.Thread(target=prices.stream_to_queue_old, args=[collection])
    liq_thread = threading.Thread(target= liquidity.activeLiq, args=[period])
    trade_thread = threading.Thread(target=trade, args=(events, strategy,execution,heartbeat))
    print "Full?:",events.full()
    trade_thread.start()
    price_thread.start()
    liq_thread.start()


import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from numpy.core.multiarray import ndarray

__author__ = "mhbashari"

class LinearRegression:
    def __init__(self, train_X: ndarray, train_Y: ndarray, learning_rate=0.001, training_epochs=100):
        self.train_X = train_X
        self.train_Y = train_Y
        self.learning_rate = learning_rate
        self.training_epochs = training_epochs

    def fit(self):
        x = tf.placeholder("float")
        y = tf.placeholder("float")
        a = tf.Variable(1.0, name="weight")
        b = tf.Variable(1.0, name="bias")

        pred = tf.multiply(x, a) + b

        cost = tf.reduce_mean(tf.abs(pred - y))

        optimizer = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(cost)

        init = tf.initialize_all_variables()

        with tf.Session() as sess:
            sess.run(init)
            for epoch in range(self.training_epochs):
                for i, out in zip(self.train_X, self.train_Y):
                    sess.run(optimizer, feed_dict={x: i, y: out})
                    print("Epoch:", '%04d' % (epoch + 1), "cost=", "W=", sess.run(a), "b=", sess.run(b))
            print("Optimization Finished!")
            training_cost = sess.run(cost, feed_dict={x: self.train_X, y: self.train_Y})
            print("Training cost=", training_cost, "a=", sess.run(a), "b=", sess.run(b), '\n')
            return sess.run(a), sess.run(b)

def visualize(a, b, train_X: ndarray, train_Y: ndarray):
    plt.plot(train_X, train_Y, 'ro', label='Original data')
    plt.plot(train_X, train_Y)
    plt.plot(train_X, a * train_X + b, label='Fitted line')
    plt.scatter(train_X, train_Y)
    plt.legend()
    plt.show()

def data_maker(num=80):
    X = np.arange(0, num, dtype=np.float32)
    Y = np.float32(np.ceil(5 * (np.sin(X) + X / 5)))
    return X, Y

if __name__ == "__main__":
    data = data_maker(5)
    regression = LinearRegression(*data_maker())
    visualize(*(regression.fit() + data_maker()))