"""
==================================
Comparing various online solvers
==================================

An example showing how different online solvers perform
on the hand-written digits dataset.

"""
# Author: Rob Zinkov <rob at zinkov dot com>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets

from sklearn.model_selection import train_test_split
from sklearn.linear_model import SGDClassifier, Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.linear_model import LogisticRegression

heldout = [0.95, 0.90, 0.75, 0.50, 0.01]
rounds = 20
digits = datasets.load_digits()
X, y = digits.data, digits.target

classifiers = [
    ("SGD", SGDClassifier()),
    ("ASGD", SGDClassifier(average=True)),
    ("Perceptron", Perceptron()),
    ("Passive-Aggressive I", PassiveAggressiveClassifier(loss='hinge',
                                                         C=1.0)),
    ("Passive-Aggressive II", PassiveAggressiveClassifier(loss='squared_hinge',
                                                          C=1.0)),
    ("SAG", LogisticRegression(solver='sag', tol=1e-1, C=1.e4 / X.shape[0]))
]

xx = 1. - np.array(heldout)

for name, clf in classifiers:
    print("training %s" % name)
    rng = np.random.RandomState(42)
    yy = []
    for i in heldout:
        yy_ = []
        for r in range(rounds):
            X_train, X_test, y_train, y_test = \
                train_test_split(X, y, test_size=i, random_state=rng)
            clf.fit(X_train, y_train)
            y_pred = clf.predict(X_test)
            yy_.append(1 - np.mean(y_pred == y_test))
        yy.append(np.mean(yy_))
    plt.plot(xx, yy, label=name)

plt.legend(loc="upper right")
plt.xlabel("Proportion train")
plt.ylabel("Test Error Rate")
plt.show()


## M-Estimators for Robust Linear Modeling

from __future__ import print_function
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

import statsmodels.api as sm

# * An M-estimator minimizes the function
#
# $$Q(e_i, \rho) = \sum_i~\rho \left (\frac{e_i}{s}\right )$$
#
# where $\rho$ is a symmetric function of the residuals
#
# * The effect of $\rho$ is to reduce the influence of outliers
# * $s$ is an estimate of scale.
# * The robust estimates $\hat{\beta}$ are computed by the iteratively re-weighted least squares algorithm

# * We have several choices available for the weighting functions to be used

norms = sm.robust.norms

def plot_weights(support, weights_func, xlabels, xticks):
    fig = plt.figure(figsize=(12,8))
    ax = fig.add_subplot(111)
    ax.plot(support, weights_func(support))
    ax.set_xticks(xticks)
    ax.set_xticklabels(xlabels, fontsize=16)
    ax.set_ylim(-.1, 1.1)
    return ax

#### Andrew's Wave

help(norms.AndrewWave.weights)

a = 1.339
support = np.linspace(-np.pi*a, np.pi*a, 100)
andrew = norms.AndrewWave(a=a)
plot_weights(support, andrew.weights, ['$-\pi*a$', '0', '$\pi*a$'], [-np.pi*a, 0, np.pi*a]);

#### Hampel's 17A

help(norms.Hampel.weights)

c = 8
support = np.linspace(-3*c, 3*c, 1000)
hampel = norms.Hampel(a=2., b=4., c=c)
plot_weights(support, hampel.weights, ['3*c', '0', '3*c'], [-3*c, 0, 3*c]);

#### Huber's t

help(norms.HuberT.weights)

t = 1.345
support = np.linspace(-3*t, 3*t, 1000)
huber = norms.HuberT(t=t)
plot_weights(support, huber.weights, ['-3*t', '0', '3*t'], [-3*t, 0, 3*t]);

#### Least Squares

help(norms.LeastSquares.weights)

support = np.linspace(-3, 3, 1000)
lst_sq = norms.LeastSquares()
plot_weights(support, lst_sq.weights, ['-3', '0', '3'], [-3, 0, 3]);

#### Ramsay's Ea

help(norms.RamsayE.weights)

a = .3
support = np.linspace(-3*a, 3*a, 1000)
ramsay = norms.RamsayE(a=a)
plot_weights(support, ramsay.weights, ['-3*a', '0', '3*a'], [-3*a, 0, 3*a]);

#### Trimmed Mean

help(norms.TrimmedMean.weights)

c = 2
support = np.linspace(-3*c, 3*c, 1000)
trimmed = norms.TrimmedMean(c=c)
plot_weights(support, trimmed.weights, ['-3*c', '0', '3*c'], [-3*c, 0, 3*c]);

#### Tukey's Biweight

help(norms.TukeyBiweight.weights)

c = 4.685
support = np.linspace(-3*c, 3*c, 1000)
tukey = norms.TukeyBiweight(c=c)
plot_weights(support, tukey.weights, ['-3*c', '0', '3*c'], [-3*c, 0, 3*c]);

#### Scale Estimators

# * Robust estimates of the location

x = np.array([1, 2, 3, 4, 500])

# * The mean is not a robust estimator of location

x.mean()

# * The median, on the other hand, is a robust estimator with a breakdown point of 50%

np.median(x)

# * Analagously for the scale
# * The standard deviation is not robust

x.std()

# Median Absolute Deviation
#
# $$ median_i |X_i - median_j(X_j)|) $$

# Standardized Median Absolute Deviation is a consistent estimator for $\hat{\sigma}$
#
# $$\hat{\sigma}=K \cdot MAD$$
#
# where $K$ depends on the distribution. For the normal distribution for example,
#
# $$K = \Phi^{-1}(.75)$$

stats.norm.ppf(.75)

print(x)

sm.robust.scale.stand_mad(x)

np.array([1,2,3,4,5.]).std()

# * The default for Robust Linear Models is MAD
# * another popular choice is Huber's proposal 2

np.random.seed(12345)
fat_tails = stats.t(6).rvs(40)

kde = sm.nonparametric.KDE(fat_tails)
kde.fit()
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)
ax.plot(kde.support, kde.density);

print(fat_tails.mean(), fat_tails.std())

print(stats.norm.fit(fat_tails))

print(stats.t.fit(fat_tails, f0=6))

huber = sm.robust.scale.Huber()
loc, scale = huber(fat_tails)
print(loc, scale)

sm.robust.stand_mad(fat_tails)

sm.robust.stand_mad(fat_tails, c=stats.t(6).ppf(.75))

sm.robust.scale.mad(fat_tails)

#### Duncan's Occupational Prestige data - M-estimation for outliers

from statsmodels.graphics.api import abline_plot
from statsmodels.formula.api import ols, rlm

prestige = sm.datasets.get_rdataset("Duncan", "car", cache=True).data

print(prestige.head(10))

fig = plt.figure(figsize=(12,12))
ax1 = fig.add_subplot(211, xlabel='Income', ylabel='Prestige')
ax1.scatter(prestige.income, prestige.prestige)
xy_outlier = prestige.ix['minister'][['income','prestige']]
ax1.annotate('Minister', xy_outlier, xy_outlier+1, fontsize=16)
ax2 = fig.add_subplot(212, xlabel='Education',
                           ylabel='Prestige')
ax2.scatter(prestige.education, prestige.prestige);

ols_model = ols('prestige ~ income + education', prestige).fit()
print(ols_model.summary())

infl = ols_model.get_influence()
student = infl.summary_frame()['student_resid']
print(student)

print(student.ix[np.abs(student) > 2])

print(infl.summary_frame().ix['minister'])

sidak = ols_model.outlier_test('sidak')
sidak.sort('unadj_p', inplace=True)
print(sidak)

fdr = ols_model.outlier_test('fdr_bh')
fdr.sort('unadj_p', inplace=True)
print(fdr)

rlm_model = rlm('prestige ~ income + education', prestige).fit()
print(rlm_model.summary())

print(rlm_model.weights)

#### Hertzprung Russell data for Star Cluster CYG 0B1 - Leverage Points

# * Data is on the luminosity and temperature of 47 stars in the direction of Cygnus.

dta = sm.datasets.get_rdataset("starsCYG", "robustbase", cache=True).data

from matplotlib.patches import Ellipse
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111, xlabel='log(Temp)', ylabel='log(Light)', title='Hertzsprung-Russell Diagram of Star Cluster CYG OB1')
ax.scatter(*dta.values.T)
# highlight outliers
e = Ellipse((3.5, 6), .2, 1, alpha=.25, color='r')
ax.add_patch(e);
ax.annotate('Red giants', xy=(3.6, 6), xytext=(3.8, 6),
            arrowprops=dict(facecolor='black', shrink=0.05, width=2),
            horizontalalignment='left', verticalalignment='bottom',
            clip_on=True, # clip to the axes bounding box
            fontsize=16,
     )
# annotate these with their index
for i,row in dta.ix[dta['log.Te'] < 3.8].iterrows():
    ax.annotate(i, row, row + .01, fontsize=14)
xlim, ylim = ax.get_xlim(), ax.get_ylim()

from IPython.display import Image
Image(filename='star_diagram.png')

y = dta['log.light']
X = sm.add_constant(dta['log.Te'], prepend=True)
ols_model = sm.OLS(y, X).fit()
abline_plot(model_results=ols_model, ax=ax)

rlm_mod = sm.RLM(y, X, sm.robust.norms.TrimmedMean(.5)).fit()
abline_plot(model_results=rlm_mod, ax=ax, color='red')

# * Why? Because M-estimators are not robust to leverage points.

infl = ols_model.get_influence()

h_bar = 2*(ols_model.df_model + 1 )/ols_model.nobs
hat_diag = infl.summary_frame()['hat_diag']
hat_diag.ix[hat_diag > h_bar]

sidak2 = ols_model.outlier_test('sidak')
sidak2.sort('unadj_p', inplace=True)
print(sidak2)

fdr2 = ols_model.outlier_test('fdr_bh')
fdr2.sort('unadj_p', inplace=True)
print(fdr2)

# * Let's delete that line

del ax.lines[-1]

weights = np.ones(len(X))
weights[X[X['log.Te'] < 3.8].index.values - 1] = 0
wls_model = sm.WLS(y, X, weights=weights).fit()
abline_plot(model_results=wls_model, ax=ax, color='green')

# * MM estimators are good for this type of problem, unfortunately, we don't yet have these yet.
# * It's being worked on, but it gives a good excuse to look at the R cell magics in the notebook.

yy = y.values[:,None]
xx = X['log.Te'].values[:,None]

get_ipython().magic(u'load_ext rmagic')

get_ipython().magic(u'R library(robustbase)')
get_ipython().magic(u'Rpush yy xx')
get_ipython().magic(u'R mod <- lmrob(yy ~ xx);')
get_ipython().magic(u'R params <- mod$coefficients;')
get_ipython().magic(u'Rpull params')

get_ipython().magic(u'R print(mod)')

print(params)

abline_plot(intercept=params[0], slope=params[1], ax=ax, color='green')

#### Exercise: Breakdown points of M-estimator

np.random.seed(12345)
nobs = 200
beta_true = np.array([3, 1, 2.5, 3, -4])
X = np.random.uniform(-20,20, size=(nobs, len(beta_true)-1))
# stack a constant in front
X = sm.add_constant(X, prepend=True) # np.c_[np.ones(nobs), X]
mc_iter = 500
contaminate = .25 # percentage of response variables to contaminate

all_betas = []
for i in range(mc_iter):
    y = np.dot(X, beta_true) + np.random.normal(size=200)
    random_idx = np.random.randint(0, nobs, size=int(contaminate * nobs))
    y[random_idx] = np.random.uniform(-750, 750)
    beta_hat = sm.RLM(y, X).fit().params
    all_betas.append(beta_hat)

all_betas = np.asarray(all_betas)
se_loss = lambda x : np.linalg.norm(x, ord=2)**2
se_beta = map(se_loss, all_betas - beta_true)

##### Squared error loss

np.array(se_beta).mean()

all_betas.mean(0)

beta_true

se_loss(all_betas.mean(0) - beta_true)


# Copyright (c) 2016 MetPy Developers.
# Distributed under the terms of the BSD 3-Clause License.
# SPDX-License-Identifier: BSD-3-Clause
"""
Find Natural Neighbors Verification
===================================

Finding natural neighbors in a triangulation

A triangle is a natural neighbor of a point if that point is within a circumradius of the
circumcenter of a circumscribed circle containing the triangle.
"""
import matplotlib.pyplot as plt
import numpy as np
from scipy.spatial import Delaunay

from metpy.gridding.triangles import find_natural_neighbors

# Create test observations, test points, and plot the triangulation and points.
gx, gy = np.meshgrid(np.arange(0, 20, 4), np.arange(0, 20, 4))
pts = np.vstack([gx.ravel(), gy.ravel()]).T
tri = Delaunay(pts)

fig, ax = plt.subplots(figsize=(15, 10))
for i, inds in enumerate(tri.simplices):
    pts = tri.points[inds]
    x, y = np.vstack((pts, pts[0])).T
    ax.plot(x, y)
    ax.annotate(i, xy=(np.mean(x), np.mean(y)))

test_points = np.array([[2, 2], [5, 10], [12, 13.4], [12, 8], [20, 20]])

for i, (x, y) in enumerate(test_points):
    ax.plot(x, y, 'k.', markersize=6)
    ax.annotate('test ' + str(i), xy=(x, y))

###########################################
# Since finding natural neighbors already calculates circumcenters and circumradii, return
# that information for later use.
#
# The key of the neighbors dictionary refers to the test point index, and the list of integers
# are the triangles that are natural neighbors of that particular test point.
#
# Since point 4 is far away from the triangulation, it has no natural neighbors.
# Point 3 is at the confluence of several triangles so it has many natural neighbors.
neighbors, tri_info = find_natural_neighbors(tri, test_points)
print(neighbors)

###########################################
# We can then use the information in tri_info later.
#
# The dictionary key is the index of a particular triangle in the Delaunay triangulation data
# structure. 'cc' is that triangle's circumcenter, and 'r' is the radius of the circumcircle
# containing that triangle.
fig, ax = plt.subplots(figsize=(15, 10))
for i, inds in enumerate(tri.simplices):
    pts = tri.points[inds]
    x, y = np.vstack((pts, pts[0])).T
    ax.plot(x, y)
    ax.annotate(i, xy=(np.mean(x), np.mean(y)))

# Using circumcenter and radius information from tri_info, plot circumcircles and
# circumcenters for each triangle.
for _idx, item in tri_info.items():
    ax.plot(item['cc'][0], item['cc'][1], 'k.', markersize=5)
    circ = plt.Circle(item['cc'], item['r'], edgecolor='k', facecolor='none',
                      transform=fig.axes[0].transData)
    ax.add_artist(circ)

ax.set_aspect('equal', 'datalim')

plt.show()


#-----------------------------------------------------------------------------
# Copyright (c) 2013-2016, PyInstaller Development Team.
#
# Distributed under the terms of the GNU General Public License with exception
# for distributing bootloader.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

# Tested with IPython 4.0.0.

from PyInstaller.compat import modname_tkinter, is_win, is_darwin
from PyInstaller.utils.hooks import collect_data_files, collect_submodules

# Ignore 'matplotlib'. IPython contains support for matplotlib.
# Ignore GUI libraries. IPython supports integration with GUI frameworks.
# Assume that it will be imported by any other module when the user really
# uses it.
excludedimports = ['gtk', 'matplotlib', 'PyQt4', 'PyQt5', 'PySide']

# IPython uses 'tkinter' for clipboard access on Linux/Unix. Exclude it on Windows and OS X.
if is_win or is_darwin:
    excludedimports.append(modname_tkinter)

datas = collect_data_files('IPython')


# pythonpath modification to make hytra and empryonic available
# for import without requiring it to be installed
from __future__ import print_function, absolute_import, nested_scopes, generators, division, with_statement, unicode_literals
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
# standard imports
from compiler.ast import flatten
import logging
import glob
import vigra
from vigra import numpy as np
import h5py
from sklearn.neighbors import KDTree
from hytra.pluginsystem.plugin_manager import TrackingPluginManager
import hytra.util.axesconversion

logger = logging.getLogger('TransitionClassifier')
logger.setLevel(logging.DEBUG)

np.seterr(all='raise')

# read in 'n2-n1' of images
def read_in_images(n1, n2, files, axes):
    gt_labelimage = [vigra.impex.readHDF5(f, 'segmentation/labels') for f in files[n1:n2]]
    gt_labelimage = [hytra.util.axesconversion.adjustOrder(img, axes, 'xyzc') for img in gt_labelimage]
    logger.info("Found segmentation of shape {}".format(gt_labelimage[0].shape))
    return gt_labelimage

# compute features from input data and return them
def compute_features(raw_image, labeled_image, n1, n2, pluginManager, filepath):
    # perhaps there is an elegant way to get into the RegionFeatureAccumulator.
    # For now, the new feature are a separate vector
    allFeat = []
    for i in range(0, n2 - n1):
        moreFeats, _ = pluginManager.applyObjectFeatureComputationPlugins(
            len(raw_image.squeeze().shape)-1, raw_image[..., i, 0], labeled_image[i][..., 0], i, filepath)
        frameFeatureItems = []
        for f in moreFeats:
            frameFeatureItems = frameFeatureItems + f.items()
        allFeat.append(dict(frameFeatureItems))
    return allFeat

# read in 'n2-n1' of labels
def read_positiveLabels(n1, n2, files):
    gt_moves = [vigra.impex.readHDF5(f, 'tracking/Moves') for f in files[n1+1:n2]]
    return gt_moves

def getValidRegionCentersAndTheirIDs(featureDict,
                                     countFeatureName='Count',
                                     regionCenterName='RegionCenter'):
    """
    From the feature dictionary of a certain frame,
    find all objects with pixel count > 0, and return their
    region centers and ids.
    """
    validObjectMask = featureDict[countFeatureName] > 0
    validObjectMask[0] = False

    regionCenters = featureDict[regionCenterName][validObjectMask, :]
    objectIds = list(np.where(validObjectMask)[0])
    return regionCenters, objectIds

def negativeLabels(features, positiveLabels):
    """
    Compute negative labels by finding 3 nearest neighbors in the next frame, and
    filtering out those pairings that are part of the positiveLabels.

    **Returns** a list of lists of pairs of indices, where there are as many inner lists
    as there are pairs of consecutive frames ordered by time,
    e.g. for frame pairs (0,1), (1,2), ... (n-1,n).
    Each pair in such a list then contains an index into the earlier frame of the pair,
    and one index into the later frame.
    """
    numFrames = len(features)
    neg_lab = []
    for i in range(1, numFrames):  # for all frames but the first
        logger.debug("Frame {}\n".format(i))
        frameNegLab = []
        # build kdtree for frame i
        centersAtI, objectIdsAtI = getValidRegionCentersAndTheirIDs(features[i])
        kdt = KDTree(centersAtI, metric='euclidean')
        # find k=3 nearest neighbors of each object of frame i-1 in frame i
        centersAtIMinusOne, objectIdsAtIMinusOne = getValidRegionCentersAndTheirIDs(features[i - 1])
        neighb = kdt.query(centersAtIMinusOne, k=3, return_distance=False)
        for j in range(0, neighb.shape[0]):  # for all valid objects in frame i-1
            logger.debug('looking at neighbors of {} at position {}'.format(
                objectIdsAtIMinusOne[j], features[i - 1]['RegionCenter'][objectIdsAtIMinusOne[j], ...]))
            for m in range(0, neighb.shape[1]):  # for all neighbors
                pair = [objectIdsAtIMinusOne[j], objectIdsAtI[neighb[j][m]]]
                if pair not in positiveLabels[i - 1].tolist():
                    # add one because we've removed the first element when creating the KD tree
                    frameNegLab.append(pair)
                    logger.debug("Adding negative example: {} at position {}".format(
                        pair, features[i]['RegionCenter'][objectIdsAtI[neighb[j][m]], ...]))
                else:
                    logger.debug("Discarding negative example {} which is a positive annotation".format(pair))
        neg_lab.append(frameNegLab)

    return neg_lab

def find_features_without_NaNs(features):
    """
    Remove all features from the list of selected features which have NaNs
    """
    selectedFeatures = features[0].keys()
    for featuresPerFrame in features:
        for key, value in featuresPerFrame.items():
            if not isinstance(value, list) and (np.any(np.isnan(value)) or np.any(np.isinf(value))):
                try:
                    selectedFeatures.remove(key)
                except:
                    pass  # has already been deleted
    forbidden = ["Global<Maximum >", "Global<Minimum >", 'Histogram', 'Polygon', 'Defect Center',
                 'Center', 'Input Center', 'Weighted<RegionCenter>']
    for f in forbidden:
        if f in selectedFeatures:
            selectedFeatures.remove(f)

    selectedFeatures.sort()
    return selectedFeatures

class TransitionClassifier:
    def __init__(self, selectedFeatures, numSamples=None):
        """
        Set up a transition classifier class that makes it easy to add samples, train and store the RF.
        :param selectedFeatures: list of feature names that are supposed to be used
        :param numSamples: if given, the data array for the samples is allocated with the proper dimensions,
                            otherwise it needs to be resized whenever new samples are added.
        """
        self.rf = vigra.learning.RandomForest()
        self.mydata = None
        self.labels = []
        self.selectedFeatures = selectedFeatures
        self._numSamples = numSamples
        self._nextIdx = 0
        # TODO: check whether prediction here and in hypotheses graph script are the same!

    def addSample(self, f1, f2, label, pluginManager):
        # if self.labels == []:
        self.labels.append(label)
        # else:
        #    self.labels = np.concatenate((np.array(self.labels),label)) # for adding batches of features
        features = self.constructSampleFeatureVector(f1, f2, pluginManager)

        if self._numSamples is None:
            # use vstack
            if self.mydata is None:
                self.mydata = features
            else:
                self.mydata = np.vstack((self.mydata, features))
        else:
            # allocate full array once, then fill in row by row
            if self.mydata is None:
                self.mydata = np.zeros((self._numSamples, features.shape[0]))

            assert(self._nextIdx < self._numSamples)
            self.mydata[self._nextIdx, :] = features
            self._nextIdx += 1

    def constructSampleFeatureVector(self, f1, f2, pluginManager):
        featVec = pluginManager.applyTransitionFeatureVectorConstructionPlugins(f1, f2, self.selectedFeatures)
        return np.array(featVec)

    # adding a comfortable function, where one can easily introduce the data
    def add_allData(self, mydata, labels):
        self.mydata = mydata
        self.labels = labels

    def train(self, withFeatureImportance=False):
        logger.info(
        "Training classifier from {} positive and {} negative labels".format(
            np.count_nonzero(np.asarray(self.labels)), len(self.labels) - np.count_nonzero(np.asarray(self.labels))))
        logger.info("Training classifier from a feature vector of length {}".format(self.mydata.shape))

        if withFeatureImportance:
            oob, featImportance = self.rf.learnRFWithFeatureSelection(
                self.mydata.astype("float32"),
                (np.asarray(self.labels)).astype("uint32").reshape(-1, 1))
            logger.debug("RF feature importance: {}".format(featImportance))
            # logger.debug('Feature names: {}'.format(self.featureNames))
        else:
            oob = self.rf.learnRF(
                self.mydata.astype("float32"),
                (np.asarray(self.labels)).astype("uint32").reshape(-1, 1))
        logger.info("RF trained with OOB Error {}".format(oob))

    # def predictSample(self, test_data=None, f1=None, f2=None):
    #     if test_data is not None:
    #         return self.rf.predictLabels(test_data.astype('float32'))
    #     else:
    #         data = self.constructSampleFeatureVector(f1, f2)
    #         if len(data.shape) < 2:
    #             data = np.expand_dims(data, axis=0)
    #         return self.rf.predictLabels(data.astype('float32'))

    # def predictProbabilities(self, test_data=None, f1=None, f2=None):
    #     if test_data is not None:
    #         return self.rf.predictProbabilities(test_data.astype('float32'))
    #     else:
    #         data = self.constructSampleFeatureVector(f1, f2)
    #         print(data)
    #         if len(data.shape) < 2:
    #             data = np.expand_dims(data, axis=0)
    #         return self.rf.predictProbabilities(data.astype('float32'))

    def predictLabels(self, test_data, threshold=0.5):
        prob = self.rf.predictProbabilities(test_data.astype('float32'))
        res = np.copy(prob)
        for i in range(0, len(prob)):
            if prob[i][1] >= threshold:
                res[i] = 1.
            else:
                res[i] = 0
        return np.delete(res, 0, 1)

    def writeRF(self, outputFilename):
        self.rf.writeHDF5(outputFilename, pathInFile='/ClassifierForests/Forest0000')

        # write selected features
        with h5py.File(outputFilename, 'r+') as f:
            featureNamesH5 = f.create_group('SelectedFeatures')
            featureNamesH5 = featureNamesH5.create_group('Standard Object Features')
            for feature in self.selectedFeatures:
                featureNamesH5.create_group(feature)

if __name__ == '__main__':
    import configargparse as argparse

    parser = argparse.ArgumentParser(description="trainRF",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-c', '--config', is_config_file=True, help='config file path')
    parser.add_argument("--groundtruth", dest='filepath', type=str, nargs='+',
                        help="read ground truth from this folder. Can be also a list of paths, to train from more datasets.", metavar="FILE")
    parser.add_argument("--groundtruth-axes", dest='groundtruth_axes', type=str, nargs='+', default=['xyzc'],
                        help="axes ordering of the ground truth segmentations per frame (no t!), e.g. xyzc", metavar="FILE")
    parser.add_argument("--raw-data-file", dest='rawimage_filename', type=str, nargs='+',
                        help="filepath+name of the raw image. Can be a list of paths, to train from more datasets.", metavar="FILE")
    parser.add_argument("--raw-data-path", dest='rawimage_h5_path', type=str,
                        help="Path inside the rawimage HDF5 file", default='volume/data')
    parser.add_argument("--raw-data-axes", dest='rawimage_axes', type=str, nargs='+', default=['txyzc'],
                        help="axes ordering of the raw image, e.g. xyztc. Can be a list of paths, to train from more datasets.", metavar="FILE")
    parser.add_argument("--init-frame", default=0, type=int, dest='initFrame',
                        help="where to begin reading the frames")
    parser.add_argument("--end-frame", default=-1, type=int, dest='endFrame',
                        help="where to end frames")
    parser.add_argument("--transition-classifier-file", dest='outputFilename', type=str,
                        help="save RF into file", metavar="FILE")
    parser.add_argument("--filepattern", dest='filepattern', type=str, nargs='+', default=['0*.h5'],
                        help="File pattern of the ground truth files. Can be also a list of paths, to train from more datasets.")
    parser.add_argument("--verbose", dest='verbose', action='store_true', default=False)
    parser.add_argument('--plugin-paths', dest='pluginPaths', type=str, nargs='+',
                        default=[os.path.abspath('../hytra/plugins')],
                        help='A list of paths to search for plugins for the tracking pipeline.')

    args, unknown = parser.parse_known_args()

    logging.basicConfig(level=logging.INFO)
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    logging.debug("Ignoring unknown parameters: {}".format(unknown))

    assert len(args.rawimage_filename) == len(args.rawimage_axes) == len(args.filepattern) == len(args.filepath) == len(args.groundtruth_axes)

    # read raw image
    numSamples = 0
    mlabels = None

    for dataset in range(len(args.rawimage_filename)):
        rawimage_filename = args.rawimage_filename[dataset]
        with h5py.File(rawimage_filename, 'r') as h5raw:
            rawimage = h5raw[args.rawimage_h5_path].value

        # transform such that the order is the following: X,Y,(Z),T, C
        rawimage = hytra.util.axesconversion.adjustOrder(rawimage, args.rawimage_axes[dataset], 'xyztc')
        logger.info('Done loading raw data from dataset {} of shape {}'.format(dataset, rawimage.shape))

        # find ground truth files
        # filepath is now a list of filepaths'
        filepath = args.filepath[dataset]
        # filepattern is now a list of filepatterns
        files = glob.glob(os.path.join(filepath, args.filepattern[dataset]))
        files.sort()
        initFrame = args.initFrame
        endFrame = args.endFrame
        if endFrame < 0:
            endFrame += len(files)

        # compute features
        trackingPluginManager = TrackingPluginManager(verbose=args.verbose,
                                                      pluginPaths=args.pluginPaths)
        features = compute_features(rawimage,
                                    read_in_images(initFrame, endFrame, files, args.groundtruth_axes[dataset]),
                                    initFrame,
                                    endFrame,
                                    trackingPluginManager,
                                    rawimage_filename)
        logger.info('Done computing features from dataset {}'.format(dataset))

        selectedFeatures = find_features_without_NaNs(features)
        pos_labels = read_positiveLabels(initFrame, endFrame, files)
        neg_labels = negativeLabels(features, pos_labels)
        numSamples += 2 * sum([len(l) for l in pos_labels]) + sum([len(l) for l in neg_labels])
        logger.info('Done extracting {} samples'.format(numSamples))
        TC = TransitionClassifier(selectedFeatures, numSamples)
        if dataset > 0:
            TC.labels = mlabels # restore labels overwritten by constructor

        # compute featuresA for each object A from the feature matrix from Vigra
        def compute_ObjFeatures(features, obj):
            featureDict = {}
            for key in features:
                if key == "Global<Maximum >" or key == "Global<Minimum >":  # this ones have only one element
                    featureDict[key] = features[key]
                else:
                    featureDict[key] = features[key][obj]
            return featureDict

        for k in range(0, len(features) - 1):
            for i in pos_labels[k]:
                # positive
                logger.debug("Adding positive sample {} from pos {} to {}".format(
                    i, features[k]['RegionCenter'][i[0]], features[k + 1]['RegionCenter'][i[1]]))
                TC.addSample(compute_ObjFeatures(
                    features[k], i[0]), compute_ObjFeatures(features[k + 1], i[1]), 1, trackingPluginManager)
                TC.addSample(compute_ObjFeatures(
                    features[k + 1], i[1]), compute_ObjFeatures(features[k], i[0]), 1, trackingPluginManager)

            for i in neg_labels[k]:
                # negative
                logger.debug("Adding negative sample {} from pos {} to {}".format(i,
                              features[k]['RegionCenter'][i[0]], features[k + 1]['RegionCenter'][i[1]]))
                TC.addSample(compute_ObjFeatures(
                    features[k], i[0]), compute_ObjFeatures(features[k + 1], i[1]), 0, trackingPluginManager)
        mlabels =TC.labels

    logger.info('Done adding samples to RF. Beginning training...')
    TC.train()
    logger.info('Done training RF')

    srcObject = compute_ObjFeatures(features[0], 1)
    destObject = compute_ObjFeatures(features[1], 2)

    # delete file before writing
    if os.path.exists(args.outputFilename):
        os.remove(args.outputFilename)
    TC.writeRF(args.outputFilename)  # writes learned RF to disk


"""
compat
======

Cross-compatible functions for Python 2 and 3.

Key items to import for 2/3 compatible code:
* iterators: range(), map(), zip(), filter(), reduce()
* lists: lrange(), lmap(), lzip(), lfilter()
* unicode: u() [u"" is a syntax error in Python 3.0-3.2]
* longs: long (int in Python 3)
* callable
* iterable method compatibility: iteritems, iterkeys, itervalues
  * Uses the original method if available, otherwise uses items, keys, values.
* types:
    * text_type: unicode in Python 2, str in Python 3
    * binary_type: str in Python 2, bytes in Python 3
    * string_types: basestring in Python 2, str in Python 3
* bind_method: binds functions to classes
* add_metaclass(metaclass) - class decorator that recreates class with with the
  given metaclass instead (and avoids intermediary class creation)

Other items:
* OrderedDefaultDict
* platform checker
"""
# pylint disable=W0611
# flake8: noqa

import functools
import itertools
from distutils.version import LooseVersion
from itertools import product
import sys
import types
from unicodedata import east_asian_width
import struct
import inspect
from collections import namedtuple

PY2 = sys.version_info[0] == 2
PY3 = (sys.version_info[0] >= 3)
PY35 = (sys.version_info >= (3, 5))
PY36 = (sys.version_info >= (3, 6))

try:
    import __builtin__ as builtins
    # not writeable when instantiated with string, doesn't handle unicode well
    from cStringIO import StringIO as cStringIO
    # always writeable
    from StringIO import StringIO
    BytesIO = StringIO
    import cPickle
    import httplib
except ImportError:
    import builtins
    from io import StringIO, BytesIO
    cStringIO = StringIO
    import pickle as cPickle
    import http.client as httplib

from pandas.compat.chainmap import DeepChainMap

if PY3:
    def isidentifier(s):
        return s.isidentifier()

    def str_to_bytes(s, encoding=None):
        return s.encode(encoding or 'ascii')

    def bytes_to_str(b, encoding=None):
        return b.decode(encoding or 'utf-8')

    # The signature version below is directly copied from Django,
    # https://github.com/django/django/pull/4846
    def signature(f):
        sig = inspect.signature(f)
        args = [
            p.name for p in sig.parameters.values()
            if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
        ]
        varargs = [
            p.name for p in sig.parameters.values()
            if p.kind == inspect.Parameter.VAR_POSITIONAL
        ]
        varargs = varargs[0] if varargs else None
        keywords = [
            p.name for p in sig.parameters.values()
            if p.kind == inspect.Parameter.VAR_KEYWORD
        ]
        keywords = keywords[0] if keywords else None
        defaults = [
            p.default for p in sig.parameters.values()
            if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
            and p.default is not p.empty
        ] or None
        argspec = namedtuple('Signature', ['args', 'defaults',
                                           'varargs', 'keywords'])
        return argspec(args, defaults, varargs, keywords)

    # have to explicitly put builtins into the namespace
    range = range
    map = map
    zip = zip
    filter = filter
    intern = sys.intern
    reduce = functools.reduce
    long = int
    unichr = chr

    # This was introduced in Python 3.3, but we don't support
    # Python 3.x < 3.4, so checking PY3 is safe.
    FileNotFoundError = FileNotFoundError

    # list-producing versions of the major Python iterating functions
    def lrange(*args, **kwargs):
        return list(range(*args, **kwargs))

    def lzip(*args, **kwargs):
        return list(zip(*args, **kwargs))

    def lmap(*args, **kwargs):
        return list(map(*args, **kwargs))

    def lfilter(*args, **kwargs):
        return list(filter(*args, **kwargs))

else:
    # Python 2
    import re
    _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")

    FileNotFoundError = IOError

    def isidentifier(s, dotted=False):
        return bool(_name_re.match(s))

    def str_to_bytes(s, encoding='ascii'):
        return s

    def bytes_to_str(b, encoding='ascii'):
        return b

    def signature(f):
        return inspect.getargspec(f)

    # import iterator versions of these functions
    range = xrange
    intern = intern
    zip = itertools.izip
    filter = itertools.ifilter
    map = itertools.imap
    reduce = reduce
    long = long
    unichr = unichr

    # Python 2-builtin ranges produce lists
    lrange = builtins.range
    lzip = builtins.zip
    lmap = builtins.map
    lfilter = builtins.filter

if PY2:
    def iteritems(obj, **kw):
        return obj.iteritems(**kw)

    def iterkeys(obj, **kw):
        return obj.iterkeys(**kw)

    def itervalues(obj, **kw):
        return obj.itervalues(**kw)

    next = lambda it: it.next()
else:
    def iteritems(obj, **kw):
        return iter(obj.items(**kw))

    def iterkeys(obj, **kw):
        return iter(obj.keys(**kw))

    def itervalues(obj, **kw):
        return iter(obj.values(**kw))

    next = next

def bind_method(cls, name, func):
    """Bind a method to class, python 2 and python 3 compatible.

    Parameters
    ----------

    cls : type
        class to receive bound method
    name : basestring
        name of method on class instance
    func : function
        function to be bound as method

    Returns
    -------
    None
    """
    # only python 2 has bound/unbound method issue
    if not PY3:
        setattr(cls, name, types.MethodType(func, None, cls))
    else:
        setattr(cls, name, func)
# ----------------------------------------------------------------------------
# functions largely based / taken from the six module

# Much of the code in this module comes from Benjamin Peterson's six library.
# The license for this library can be found in LICENSES/SIX and the code can be
# found at https://bitbucket.org/gutworth/six

# Definition of East Asian Width
# http://unicode.org/reports/tr11/
# Ambiguous width can be changed by option
_EAW_MAP = {'Na': 1, 'N': 1, 'W': 2, 'F': 2, 'H': 1}

if PY3:
    string_types = str,
    integer_types = int,
    class_types = type,
    text_type = str
    binary_type = bytes

    def u(s):
        return s

    def u_safe(s):
        return s

    def strlen(data, encoding=None):
        # encoding is for compat with PY2
        return len(data)

    def east_asian_len(data, encoding=None, ambiguous_width=1):
        """
        Calculate display width considering unicode East Asian Width
        """
        if isinstance(data, text_type):
            return sum([_EAW_MAP.get(east_asian_width(c), ambiguous_width) for c in data])
        else:
            return len(data)

    def import_lzma():
        """ import lzma from the std library """
        import lzma
        return lzma

    def set_function_name(f, name, cls):
        """ Bind the name/qualname attributes of the function """
        f.__name__ = name
        f.__qualname__ = '{klass}.{name}'.format(
            klass=cls.__name__,
            name=name)
        f.__module__ = cls.__module__
        return f

    ResourceWarning = ResourceWarning

else:
    string_types = basestring,
    integer_types = (int, long)
    class_types = (type, types.ClassType)
    text_type = unicode
    binary_type = str

    def u(s):
        return unicode(s, "unicode_escape")

    def u_safe(s):
        try:
            return unicode(s, "unicode_escape")
        except:
            return s

    def strlen(data, encoding=None):
        try:
            data = data.decode(encoding)
        except UnicodeError:
            pass
        return len(data)

    def east_asian_len(data, encoding=None, ambiguous_width=1):
        """
        Calculate display width considering unicode East Asian Width
        """
        if isinstance(data, text_type):
            try:
                data = data.decode(encoding)
            except UnicodeError:
                pass
            return sum([_EAW_MAP.get(east_asian_width(c), ambiguous_width) for c in data])
        else:
            return len(data)

    def import_lzma():
        """ import the backported lzma library
        or raise ImportError if not available """
        from backports import lzma
        return lzma

    def set_function_name(f, name, cls):
        """ Bind the name attributes of the function """
        f.__name__ = name
        return f

    class ResourceWarning(Warning):
        pass

string_and_binary_types = string_types + (binary_type,)

try:
    # callable reintroduced in later versions of Python
    callable = callable
except NameError:
    def callable(obj):
        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)

def add_metaclass(metaclass):
    """Class decorator for creating a class with a metaclass."""
    def wrapper(cls):
        orig_vars = cls.__dict__.copy()
        orig_vars.pop('__dict__', None)
        orig_vars.pop('__weakref__', None)
        for slots_var in orig_vars.get('__slots__', ()):
            orig_vars.pop(slots_var)
        return metaclass(cls.__name__, cls.__bases__, orig_vars)
    return wrapper

from collections import OrderedDict, Counter

if PY3:
    def raise_with_traceback(exc, traceback=Ellipsis):
        if traceback == Ellipsis:
            _, _, traceback = sys.exc_info()
        raise exc.with_traceback(traceback)
else:
    # this version of raise is a syntax error in Python 3
    exec("""
def raise_with_traceback(exc, traceback=Ellipsis):
    if traceback == Ellipsis:
        _, _, traceback = sys.exc_info()
    raise exc, None, traceback
""")

raise_with_traceback.__doc__ = """Raise exception with existing traceback.
If traceback is not passed, uses sys.exc_info() to get traceback."""

# http://stackoverflow.com/questions/4126348
# Thanks to @martineau at SO

from dateutil import parser as _date_parser
import dateutil
if LooseVersion(dateutil.__version__) < '2.0':
    @functools.wraps(_date_parser.parse)
    def parse_date(timestr, *args, **kwargs):
        timestr = bytes(timestr)
        return _date_parser.parse(timestr, *args, **kwargs)
elif PY2 and LooseVersion(dateutil.__version__) == '2.0':
    # dateutil brokenness
    raise Exception('dateutil 2.0 incompatible with Python 2.x, you must '
                    'install version 1.5 or 2.1+!')
else:
    parse_date = _date_parser.parse

class OrderedDefaultdict(OrderedDict):

    def __init__(self, *args, **kwargs):
        newdefault = None
        newargs = ()
        if args:
            newdefault = args[0]
            if not (newdefault is None or callable(newdefault)):
                raise TypeError('first argument must be callable or None')
            newargs = args[1:]
        self.default_factory = newdefault
        super(self.__class__, self).__init__(*newargs, **kwargs)

    def __missing__(self, key):
        if self.default_factory is None:
            raise KeyError(key)
        self[key] = value = self.default_factory()
        return value

    def __reduce__(self):  # optional, for pickle support
        args = self.default_factory if self.default_factory else tuple()
        return type(self), args, None, None, list(self.items())

# https://github.com/pandas-dev/pandas/pull/9123
def is_platform_little_endian():
    """ am I little endian """
    return sys.byteorder == 'little'

def is_platform_windows():
    return sys.platform == 'win32' or sys.platform == 'cygwin'

def is_platform_linux():
    return sys.platform == 'linux2'

def is_platform_mac():
    return sys.platform == 'darwin'

def is_platform_32bit():
    return struct.calcsize("P") * 8 < 64


# coding: utf-8

# In[ ]:

import h2o
import pandas as pd

# initialize the model scoring server
h2o.init(nthreads=1,max_mem_size=1, start_h2o=True, strict_version_check = False)

def approve_loan(Loan_Amount,Term,Interest_Rate,Employment_Years,Home_Ownership,Annual_Income,Verification_Status,Loan_Purpose,State,
                 Debt_to_Income,Delinquent_2yr,Revolving_Cr_Util,Total_Accounts,Longest_Credit_Length):
    # connect to the model scoring service
    h2o.connect()

    # open the downloaded model
    ChurnPredictor = h2o.load_model(path='DRF_model_1496459915419_4')

    # define a feature vector to evaluate with the model
    newData = pd.DataFrame({'Loan_Amount' : Loan_Amount,
                            'Term' : Term,
                            'Interest_Rate' : Interest_Rate,
                            'Employment_Years' : Employment_Years,
                            'Home_Ownership' : Home_Ownership,
                            'Annual_Income' : Annual_Income,
                            'Verification_Status' : Verification_Status,
                            'Loan_Purpose' : Loan_Purpose,
                            'State' : State,
                            'Debt_to_Income' : Debt_to_Income,
                            'Delinquent_2yr' : Delinquent_2yr,
                            'Revolving_Cr_Util' : Revolving_Cr_Util,
                            'Total_Accounts' : Total_Accounts,
                            'Longest_Credit_Length' : Longest_Credit_Length}, index=[0])

    # evaluate the feature vector using the model
    predictions = ChurnPredictor.predict(h2o.H2OFrame(newData))
    predictionsOut = h2o.as_list(predictions, use_pandas=False)
    prediction = predictionsOut[1][0]
    probabilityBad = predictionsOut[1][1]
    probabilityGood = predictionsOut[1][2]
    return "Prediction: " + str(prediction) + " |Probability of Bad Loan: " + str(probabilityBad) + " |Probability of Good Loan: " + str(probabilityGood)


# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import json
import csv
import copy
import pandas as pd

CENSUS_DATA_COLUMN_START = 7

class CensusGenericDataLoaderBase(object):
    GENERIC_TEMPLATE_STAT_VAR = """Node: {StatisticalVariable}
description: "{description}"
typeOf: dcs:StatisticalVariable
populationType: schema:Person
measuredProperty: {measuredProperty}

"""

    GENERIC_TEMPLATE_TMCF = """Node: E:IndiaCensus{year}_{dataset_name}->E0
typeOf: dcs:StatVarObservation
variableMeasured: C:IndiaCensus{year}_{dataset_name}->StatisticalVariable
observationDate: C:IndiaCensus{year}_{dataset_name}->Year
observationAbout: E:IndiaCensus{year}_{dataset_name}->E1
value: C:IndiaCensus{year}_{dataset_name}->Value

Node: E:IndiaCensus{year}_{dataset_name}->E1
typeOf: schema:Place
indianCensusAreaCode{year}: C:IndiaCensus{year}_{dataset_name}->census_location_id"""
    """An object that represents Census Data and its variables.

    Attributes:
        census_columns (list): It will have all the data column names of a dataset
        census_year : Census year
        csv_file_path : Path where cleaned csv file will be saved
        data_file_path : Input XLS file from Census of India. Can be url or local path.
        dataset_name : Census dataset name. Eg:Primary_Abstract_Data
        existing_stat_var (list): List of existing stat vars that we don't need to generate
        mcf (list): Description
        mcf_file_path : Description
        metadata_file_path : Description
        raw_df : Raw census data as dataframe
        stat_var_index (dict): local storage for census column name and corresponding statvar
        tmcf_file_path : Path where generated tmcf file will be saved
    """

    def __init__(self, data_file_path, metadata_file_path, mcf_file_path,
                 tmcf_file_path, csv_file_path, existing_stat_var, census_year,
                 dataset_name):
        """
        Constructor

        Args:
            data_file_path :  Input XLS file from Census of India. Can be url or local path
            metadata_file_path : Meta data csv file which has attribute details
            mcf_file_path : Path where generated mcf file will be saved
            tmcf_file_path : Path where generated tmcf file will be saved
            csv_file_path : Path where cleaned csv file will be saved
            existing_stat_var : List of existing stat vars that we don't need to generate
            census_year : Census Year
            dataset_name : Census dataset name. Eg:Primary_Abstract_Data
        """
        self.data_file_path = data_file_path
        self.metadata_file_path = metadata_file_path
        self.mcf_file_path = mcf_file_path
        self.csv_file_path = csv_file_path
        self.tmcf_file_path = tmcf_file_path
        self.existing_stat_var = existing_stat_var
        self.census_year = census_year
        self.dataset_name = dataset_name
        self.raw_df = None
        self.stat_var_index = {}
        self.census_columns = []

    def _download_and_standardize(self):
        dtype = {
            'State': str,
            'District': str,
            'Subdistt': str,
            "Town/Village": str
        }
        self.raw_df = pd.read_excel(self.data_file_path, dtype=dtype)
        self.census_columns = self.raw_df.columns[CENSUS_DATA_COLUMN_START:]

    def _format_location(self, row):
        # In this specific format there is no Level defined.
        # A non zero location code from the lowest administration area
        # takes the precedence.
        if row["Town/Village"] != "000000":
            return row["Town/Village"]

        elif row["Subdistt"] != "00000":
            return row["Subdistt"]

        elif row["District"] != "000":
            return row["District"]

        elif row["State"] != "00":
            return row["State"]
        else:
            # This is india level location
            return "0"

    def _format_data(self):
        # This function is overridden in the child class
        pass

    def _get_base_name(self, row):
        # This function is overridden in the child class
        name = "Count_"
        return name

    def _create_variable(self, data_row, **kwargs):
        # This function is overridden in the child class
        pass

    def _create_mcf(self):
        # This function is overridden in the child class
        pass

    def _create_tmcf(self):
        with open(self.tmcf_file_path, 'w+', newline='') as f_out:
            f_out.write(
                self.GENERIC_TEMPLATE_TMCF.format(
                    year=self.census_year, dataset_name=self.dataset_name))

    def process(self):
        self._download_and_standardize()
        self._create_mcf()
        self._create_tmcf()
        self._format_data()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches
x, y = np.random.rand(2, 200)

fig, ax = plt.subplots()
circ = patches.Circle((0.5, 0.5), 0.25, alpha=0.8, fc='yellow')
ax.add_patch(circ)

ax.plot(x, y, alpha=0.2)
line, = ax.plot(x, y, alpha=1.0, clip_path=circ)

class EventHandler(object):
    def __init__(self):
        fig.canvas.mpl_connect('button_press_event', self.onpress)
        fig.canvas.mpl_connect('button_release_event', self.onrelease)
        fig.canvas.mpl_connect('motion_notify_event', self.onmove)
        self.x0, self.y0 = circ.center
        self.pressevent = None

    def onpress(self, event):
        if event.inaxes != ax:
            return

        if not circ.contains(event)[0]:
            return

        self.pressevent = event

    def onrelease(self, event):
        self.pressevent = None
        self.x0, self.y0 = circ.center

    def onmove(self, event):
        if self.pressevent is None or event.inaxes != self.pressevent.inaxes:
            return

        dx = event.xdata - self.pressevent.xdata
        dy = event.ydata - self.pressevent.ydata
        circ.center = self.x0 + dx, self.y0 + dy
        line.set_clip_path(circ)
        fig.canvas.draw()

handler = EventHandler()
plt.show()


"""
MIT License

Copyright (c) 2016 Zeke Barge

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import os
import pandas as pd
import logging
from functools import partial
from zeex.core.compat import QtGui, QtCore
from zeex.core.models.actions import FileViewModel
from zeex.core.ctrls.dataframe import DataFrameModelManager
from zeex.core.ui.actions.merge_purge_ui import Ui_MergePurgeDialog
from zeex.core.utility.collection import DictConfig, SettingsINI
from zeex.core.utility.pandatools import gather_frame_fields
from zeex.core.utility.widgets import create_standard_item_model
from zeex.core.views.basic.map_grid import MapGridDialog
from zeex.core.views.basic.push_grid import PushGridHandler
from zeex.core.ctrls.dataframe import DataFrameModel

class MergePurgeDialog(QtGui.QDialog, Ui_MergePurgeDialog):
    """
    This dialog allows a user to do large updates on a given source DataFrameModel.
        - Merging other file(s) with the source based on common keys/fields
        - Purging records from the source using other file(s) based on common keys/fields
        - Sorting the DataFrame by multiple columns/ascending/descending
        - Deduplicating the DataFrame based on common keys/fields
    Settings can exported to a config.ini file and re-imported at a later time.

    """
    signalMergeFileOpened = QtCore.Signal(str) # file path
    signalSFileOpened = QtCore.Signal(str) # file path
    signalSourcePathSet = QtCore.Signal(str) #file path
    signalExecuted = QtCore.Signal(str, str, str) # source_path, dest_path, report_path

    def __init__(self, df_manager: DataFrameModelManager, parent=None, source_model=None):
        """
        :param df_manager: (DataFrameModelManager)
            This will be used to handle reading/updating of DataFrameModels used
            in the operation.
        :param parent: (QMainWindow)

        :param source_model: (DataFrameModel)
            An optional source DataFrameModel
        """
        self.df_manager = df_manager
        QtGui.QDialog.__init__(self, parent)
        self.setupUi(self)
        self.source_model = source_model
        self._merge_view_model = FileViewModel()
        self._suppress_view_model = FileViewModel()
        self._purge_files = {}
        self._merge_files = {}
        self._field_map_grids = {}
        self._field_map_data = {}
        self.sortAscHandler = None
        self.sortOnHandler = None
        self.dedupeOnHandler = None
        self.uniqueFieldsHandler = None
        self.gatherFieldsHandler = None
        self.configure()
        if self.source_model is not None:
            self.set_source_model(source_model, configure=True)

    def configure(self, source_path=None, dest_path=None):
        """
        Connects main buttons and actions.
        :param source_path: (str, default None)
            If this is None there must be a valid path already in the sourcePathLineEdit or an AssertionError raises.

        :param dest_path: (str, default None)
            Optional custom destination path to be added to the destPathLineEdit.
        :return: None
        """
        if source_path is None:
            source_path = self.sourcePathLineEdit.text()
        if os.path.isfile(source_path):
            self.set_line_edit_paths(source_path, dest_path=dest_path)
        if self.sortAscHandler is None:
            self.set_handler_sort_asc()
        source_func = partial(self.open_file, model_signal=self.signalSourcePathSet)

        self.signalSourcePathSet.connect(self.set_source_model_from_browse)
        self.btnBrowseSourcePath.clicked.connect(source_func)
        self.btnBrowseDestPath.clicked.connect(self.set_dest_path_from_browse)
        self.signalMergeFileOpened.connect(self.add_merge_file)
        merge_file_func = partial(self.open_file, model_signal=self.signalMergeFileOpened)
        self.btnAddMergeFile.clicked.connect(merge_file_func)
        self.btnBrowseMergeFile.clicked.connect(merge_file_func)
        self.btnDeleteMergeFile.clicked.connect(partial(self.remove_file, self.mergeFileTable))
        self.btnEditMergeFile.clicked.connect(partial(self.open_edit_file_window, self.mergeFileTable, self._merge_files))
        self.mergeFileTable.setModel(self._merge_view_model)

        self.signalSFileOpened.connect(self.add_purge_file)
        sfile_func = partial(self.open_file, model_signal=self.signalSFileOpened)
        self.btnEditSFile.clicked.connect(partial(self.open_edit_file_window, self.sFileTable, self._purge_files))
        self.btnDeleteSFile.clicked.connect(partial(self.remove_file, self.sFileTable))
        self.btnAddSFile.clicked.connect(sfile_func)
        self.btnBrowseSFile.clicked.connect(sfile_func)
        self.sFileTable.setModel(self._suppress_view_model)
        self.btnMapSFields.clicked.connect(partial(self.open_field_map, self.sFileTable, self._purge_files))
        self.btnMapMergeFields.clicked.connect(partial(self.open_field_map, self.mergeFileTable, self._merge_files))
        self.btnExecute.clicked.connect(self.execute)
        self.btnExportTemplate.clicked.connect(self.export_settings)
        self.btnImportTemplate.clicked.connect(self.import_settings)
        self.btnReset.clicked.connect(self.reset)

    def set_source_model_from_browse(self, filepath):
        self.set_line_edit_paths(filepath, dest_path=False)
        self.set_source_model(configure=True)

    def set_dest_path_from_browse(self, filepath=None):
        if filepath is None:
            try:
                dirname = os.path.dirname(self.df_manager.last_path_read)
            except:
                dirname = ''
            filepath = QtGui.QFileDialog.getOpenFileName(self, dir=dirname)[0]
        self.destPathLineEdit.setText(filepath)

    def set_source_model(self, model=None, configure=True):
        """
        Sets the source DataFrameModel for the Dialog.

        :param model: (DataFrameModel)
            The DataFrameModel to be set.
        :param configure:
            True re-configures file path line edits and the listviews.
        :return:
        """
        if not hasattr(model, 'dataFrame'):
            if model is None:
                model = self.sourcePathLineEdit.text()
            if isinstance(model, str) and os.path.exists(model):
                model = self.df_manager.read_file(model)
            else:
                raise Exception("model parameter must be a filepath or a qtpandas.models.DataFrameModel")

        if self.source_model is not None:
            models_different = model.filePath != self.source_model.filePath
            if models_different:
                try:
                    self.source_model.dataFrameChanged.disconnect(self.sync)
                except RuntimeError:
                    pass
        else:
            models_different = True

        if models_different:
            self.source_model = model
            self.source_model.dataFrameChanged.connect(self.sync)

        if configure:
            self.sync()

    def sync(self):

        df = self.source_model.dataFrame()
        cols = df.columns.tolist()

        if self.dedupeOnHandler is None or self.uniqueFieldsHandler is None:
            self.set_push_grid_handlers()
        else:
            self.dedupeOnHandler.set_model_from_list(cols)
            self.gatherFieldsHandler.set_model_from_list(cols)
            self.sortOnHandler.set_model_from_list(cols)
            self.uniqueFieldsHandler.set_model_from_list(cols)

        self.set_primary_key_combo_box()
        self.set_line_edit_paths(source_path=self.source_model.filePath)

    def set_line_edit_paths(self, source_path=None, dest_path=None):
        """
        Sets the source/destination line edits in the Dialog.
        :param source_path: (str, default None)
            An optional valid filepath for the source DataFrameModel.
            If None, :param dest_path cannot be None.
        :param dest_path: (str, default None)
            An optional destination path. One will be created automatically
            if None is given.
            False will prevent the destination path from being set at all.
        :return: None
        """
        assert any([dest_path, source_path]), "source_path or dest_path must be set."

        if dest_path is None:
            dirname = os.path.dirname(source_path)
            base, ext = os.path.splitext(os.path.basename(source_path))
            dest_path = os.path.join(dirname, base + "_merged" + ext)

        if source_path:
            self.sourcePathLineEdit.setText(source_path)

        if dest_path:
            self.destPathLineEdit.setText(dest_path)

    def set_push_grid_handlers(self, column_model=None, sorton_model=None, sortasc_model=None,
                               dedupe_model=None, gather_model=None, unique_model=None):
        """
        Sets all default push grid handlers for the dialog.

        :param column_model: (QStandardItemModel, default None)
        :param sorton_model:  ((QStandardItemModel,list) default None)
        :param sortasc_model: ((QStandardItemModel,list) default None)
        :param dedupe_model: ((QStandardItemModel,list) default None)
        :return:
        """

        if column_model is None:
            column_model = self.get_source_columns_model()

        self.set_handler_sort_on(column_model=None, default_model=sorton_model)
        self.set_handler_sort_asc(default_model=sortasc_model)
        self.set_handler_dedupe_on(column_model=None, default_model=dedupe_model)
        self.set_handler_gather_fields(column_model=None, default_model=gather_model)
        self.set_handler_unique_fields(column_model=None, default_model=unique_model)

    def set_handler_sort_on(self, column_model=None, default_model=None):
        if column_model is None:
            column_model = self.get_source_columns_model()
        self.sortOnHandler = PushGridHandler(left_model=column_model, left_view=self.sortOnLeftView,
                                             left_button=self.sortOnLeftButton,
                                             left_delete=True, right_model=default_model,
                                             right_view=self.sortOnRightView,
                                             right_button=self.sortOnRightButton)

    def set_handler_sort_asc(self, default_model=None, overwrite=False):
        if self.sortAscHandler is None or default_model is not None or overwrite:
            sort_asc = QtGui.QStandardItemModel()
            sort_asc.appendRow(QtGui.QStandardItem('True'))
            sort_asc.appendRow(QtGui.QStandardItem('False'))
            self.sortAscHandler = PushGridHandler(left_model=sort_asc, left_view=self.sortAscLeftView,
                                                  left_button=self.sortAscLeftButton,
                                                  left_delete=False, right_model=default_model,
                                                  right_view=self.sortAscRightView,
                                                  right_button=self.sortAscRightButton)

    def set_handler_dedupe_on(self, column_model=None, default_model=None):
        if column_model is None:
            column_model = self.get_source_columns_model()
        self.dedupeOnHandler = PushGridHandler(left_model=column_model, left_view=self.dedupeOnLeftView,
                                               left_button=self.dedupeOnLeftButton,
                                               left_delete=True, right_model=default_model,
                                               right_view=self.dedupeOnRightView,
                                               right_button=self.dedupeOnRightButton)

    def set_handler_gather_fields(self, column_model=None, default_model=None):
        if column_model is None:
            column_model = self.get_source_columns_model()
        self.gatherFieldsHandler = PushGridHandler(left_model=column_model,
                                                   left_view=self.gatherFieldsListViewLeft,
                                                   left_button=self.gatherFieldsButtonLeft,
                                                   left_delete=True, right_model=default_model,
                                                   right_view=self.gatherFieldsListViewRight,
                                                   right_button=self.gatherFieldsButtonRight)

    def set_handler_unique_fields(self, column_model=None, default_model=None):
        if column_model is None:
            column_model = self.get_source_columns_model()
        self.uniqueFieldsHandler = PushGridHandler(left_model=column_model,
                                                   left_view=self.uniqueFieldsListViewLeft,
                                                   left_button=self.uniqueFieldsPushButtonLeft,
                                                   left_delete=True, right_model=default_model,
                                                   right_view=self.uniqueFieldsListViewRight,
                                                   right_button=self.uniqueFieldsPushButtonRight)

    def get_source_columns_model(self, raise_on_error=True) -> QtGui.QStandardItemModel:
        """
        Quick way to get a QStandardItemModel form the DataFrameModel's columns.
        :param raise_on_error: (bool, default True)
            Raises an error if the source_model has not yet been set.
        :return: (QtGui.QStandardItemModel)
        """
        if self.source_model is None:
            if raise_on_error:
                raise Exception("Cannot get source_columns as source_model is None!")
            else:
                columns = []
        else:
            columns = self.source_model.dataFrame().columns.tolist()

        return create_standard_item_model(columns)

    def open_file(self, file_names: list=None, model_signal=None, allow_multi=True):
        """
        Opens a Merge or Purge file (or really any file) and calls the
        given model signal after registering the DataFrameModel with the DataFrameModelManager.
        :param file_names: (list, default None)
            An optional list of filenames to open.
            The user must select filenames otherwise.
        :param model_signal: (QtCore.Signal)
            A signal to be called after successfully reading the DataFrameModel.
        :param allow_multi: (bool, default True)
            True allows multiple files to be read (and the signal called each time).
            False allows only the first file to be read.
        :return: None
            You can call MergePurgeDialog.df_manager.get_frame(filename) to
            retrieve a DataFrameModel.
        """
        if file_names is None:
            dirname = os.path.dirname(self.sourcePathLineEdit.text())
            file_names = QtGui.QFileDialog.getOpenFileNames(parent=self,
                                                            dir=dirname)[0]

        if isinstance(file_names, str):
            file_names = list(file_names)

        assert not isinstance(file_names, str) and hasattr(file_names, "__iter__"), "file_names is not list-like!"

        if allow_multi is False:
            file_names = list(file_names[0])

        for f in file_names:
            try:
                if not isinstance(f, str) and hasattr(f, '__iter__'):
                    f = f[0]
                if os.path.exists(f):
                    self.df_manager.read_file(f)
                    if model_signal is not None:
                        model_signal.emit(f)
                        logging.info("Emitted signal: {}".format(f))
            except Exception as e:
                logging.error(e)

    @QtCore.Slot(str)
    def add_merge_file(self, file_path):
        """
        Adds a merge file to the merge view and
        also updates the internal dictionary storing the filepath/model.
        :param file_path: (str)
            The file path to add.
        :return: None
        """
        model = self.df_manager.get_model(file_path)
        model.enableEditing(True)
        self._merge_files.update({file_path:model})
        self._merge_view_model.append_df_model(model)
        self.mergeFileTable.setColumnWidth(0, 500)
        self._merge_view_model.setHorizontalHeaderLabels(['filepath', 'count'])

    @QtCore.Slot(str)
    def add_purge_file(self, file_path):
        """
        Adds a purge file to the purge view and
        also updates the internal dictionary storing the filepath/model.
        :param file_path: (str)
            The file path to add.
        :return: None
        """
        model = self.df_manager.get_model(file_path)
        model.enableEditing(True)
        self._purge_files.update({file_path:model})
        self._suppress_view_model.append_df_model(model)
        self.sFileTable.setColumnWidth(0, 500)
        self._suppress_view_model.setHorizontalHeaderLabels(['filepath', 'count'])

    def remove_file(self, view, indexes=None):
        """
        Removes selected file(s) from the given view.
        :param view: (QListView)
            The view to drop the selected indexes on.
        :param indexes: (list, default None)
            A list of given indexes to drop.
            Otherwise relies on selected indexes in the view.
        :return: None
        """
        if indexes is None:
            indexes = [x.row() for x in view.selectedIndexes()]
        model = view.model()
        for idx in indexes:
            model.takeRow(idx)

    def open_field_map(self, view, models):
        """
        Connects a MapGridDialog to help the  user map field names that
        are different between the source DataFrameModel and the
        selected merge or suppression DataFrameModel.

        :param view: (QtGui.QTableView)
            The view that has a selected filepath
        :param models: (dict)
            The dictionary of {file_path:DataFrameModel} where
            dataframe columns can be gathered from.
        :return: None

        """
        idx = view.selectedIndexes()[0]
        view_model = view.model()
        view_item = view_model.item(idx.row())
        view_item_text = view_item.text()

        try:
            self._field_map_grids[view_item_text].show()
        except KeyError:
            dfmodel = models[view_item_text]
            colmodel = dfmodel._dataFrame.columns.tolist()

            if self.source_model is None:
                self.set_source_model()

            source_colmodel = self.source_model._dataFrame.columns.tolist()

            fmap = MapGridDialog(parent=self)
            fmap.load_combo_box(source_colmodel, left=True)
            fmap.load_combo_box(colmodel, left=False)
            fmap.setWindowTitle("Map Fields")
            fmap.labelLeft.setText(os.path.basename(self.source_model.filePath))
            fmap.labelRight.setText(os.path.basename(dfmodel.filePath))
            fmap.signalNewMapping.connect(lambda x: self._field_map_data.update({dfmodel.filePath: x}))

            self._field_map_grids[view_item_text] = fmap
            self._field_map_grids[view_item_text].show()

    def get_map_grid(self, file_path):
        """
        Accessor to the MergePurgeDialog._field_map_grids dictionary.
        Contains map grid dialogs.
        :param file_path: (str)
            The filepath related to the desired MapGridDialog.
        :return: (MapGridDialog, None)
        """
        return self._field_map_grids.get(file_path, None)

    def open_edit_file_window(self, view, models):
        """
        Connects a DataFrameModel selected in the view
        to a FileTableWindow where the model can be edited.

        :param view: (QtGui.QTableView)
            The view that has a selected filepath
        :param models: (dict)
            The dictionary of {file_path:DataFrameModel}
            to supply the FileTableWindow
        :return: None
        """
        try:
            idx = view.selectedIndexes()[0]
        except IndexError:
            raise IndexError("No file selected to open.")
        vmodel = view.model()
        vitem = vmodel.item(idx.row())
        model = models.get(vitem.text())

        fp = model.filePath
        wdw = self.df_manager.get_fileview_window(fp)
        # Prevent wierdos from doing an endless loop of MergePurge windows.
        # That would be pretty funny, though..
        wdw.actionMergePurge.setVisible(False)

        wdw.show()

    def execute(self):
        """
        Executes the merge_purge based upon the given settings.
        :return: None
        """
        if self.source_model is None:
            self.set_source_model()

        suppressed_results = {}
        merged_results = {}
        source_path = self.sourcePathLineEdit.text()
        dest_path = self.destPathLineEdit.text()
        source_df = self.source_model.dataFrame().copy()
        source_df.loc[:, 'ORIG_IDXER'] = source_df.index
        source_size = source_df.index.size
        index_label = self.primaryKeyComboBox.currentText()

        sort_on = self.sortOnHandler.get_model_list(left=False)
        ascending = self.sortAscHandler.get_model_list(left=False)
        dedupe_on = self.dedupeOnHandler.get_model_list(left=False)
        gather_fields = self.gatherFieldsHandler.get_model_list(left=False)
        overwrite_existing = self.gatherFieldsOverWriteCheckBox.isChecked()

        # Make sure ascending/sort_on lists are equal.
        while len(sort_on) < len(ascending):
            ascending.append(False)

        while len(sort_on) > len(ascending):
            ascending.pop()

        # Get all merge models and merge.
        # Absorb all rows and columns
        for file_path, merge_model in self._merge_files.items():
            pre_size = source_df.index.size
            other_df = merge_model.dataFrame()
            if gather_fields:
                assert index_label in other_df.columns, "DataFrameModel for {} missing column {}".format(
                                                         merge_model.filePath, index_label)
                source_df = gather_frame_fields(source_df, other_df, index_label=index_label,
                                                fields=gather_fields, copy_frames=True,
                                                append_missing=True, overwrite=overwrite_existing)
            else:
                source_df = pd.concat([source_df, other_df])
            merged_results.update({merge_model.filePath: source_df.index.size - pre_size})
        # Get all suppression models and suppress.
        for file_path, suppress_model in self._purge_files.items():
            map_dict = self._field_map_data.get(file_path, {})
            sframe = suppress_model.dataFrame().copy()
            sframe.drop(['ORIG_IDXER'], axis=1, inplace=True, errors='ignore')

            if map_dict:
                # A mapping exists - rename the data and get the key_cols
                key_cols = list(map_dict.values())
                sframe.rename(columns=map_dict, inplace=True)
            else:
                # No mapping exists - Try to use the dedupe_on cols as key_cols
                key_cols = dedupe_on.copy()
                missing = [x for x in key_cols if x not in sframe.columns]
                if missing:
                    raise KeyError("Suppression file {} must have a field mapping or \
                                    have the dedupe column labels, it has neither!.".format(
                                    suppress_model.filePath))

            sframe = sframe.loc[:, key_cols].drop_duplicates(key_cols)
            badframe = pd.merge(source_df, sframe, how='inner', left_on=key_cols, right_on=key_cols)
            source_df = source_df.loc[~source_df.index.isin(badframe.loc[:, 'ORIG_IDXER'].tolist()), :]
            suppressed_results.update({suppress_model.filePath: badframe.index.size})

        # Sort the data
        if sort_on and ascending:
            source_df.sort_values(sort_on, ascending=ascending, inplace=True)

        # Deduplicate the data.
        if dedupe_on:
            pre_size = source_df.index.size
            source_df.drop_duplicates(dedupe_on, inplace=True)
            dedupe_lost = pre_size - source_df.index.size
        else:
            dedupe_lost = 0

        # Export the data - done!
        source_df.drop(['ORIG_IDXER'], axis=1, inplace=True, errors='ignore')
        source_df.to_csv(dest_path, index=False)
        logging.info("Exported: {}".format(dest_path))

        merge_string = "\n".join("Gained {} merging {}".format(v, k) for k, v in merged_results.items())
        suppress_string = "\n".join("Lost {} suppressing {}".format(v, k) for k,v in suppressed_results.items())
        report = """
        Merge Purge Report
        ==================
        Original Size: {}
        Final Size: {}
        Source Path: {}
        Output Path: {}

        Merge:
        ==================
        {}

        Purge:
        ==================
        {}

        Sort:
        ==================
            SORT BY: {}
            SORT ASCENDING: {}

        Dedupe:
        ==================
            DEDUPE ON: {}
            RECORDS LOST: {}

        """.format(source_size, source_df.index.size, source_path,
                   dest_path, merge_string, suppress_string,
                   sort_on, ascending, dedupe_on, dedupe_lost)

        report_path = os.path.splitext(dest_path)[0] + "_report.txt"
        with open(report_path, "w") as fh:
            fh.write(report)

        self.signalExecuted.emit(source_path, dest_path, report_path)

    def get_settings(self, dc:DictConfig = None, section="MERGE_PURGE") -> DictConfig:
        """
        Gathers the settings out of the Dialog and
        returns a DictConfig object with updated settings.

        :param dc (DictConfig, default None)
            An optional DictConfig object, one is created if none is given.
        :param section (str, default 'MERGE_PURGE')
            An optional section name to apply settings to.
            A pre-existing section with this name would be overwritten.
        :return: (DictConfig)
            An updated DictConfig object.
        """
        if dc is None:
            dc = DictConfig()
        if dc.has_section(section):
            dc.remove_section(section)
        dc.add_section(section)

        dc.set_safe(section, 'source_path',           self.sourcePathLineEdit.text())
        dc.set_safe(section, 'dest_path',             self.destPathLineEdit.text())
        dc.set_safe(section, 'primary_key',           self.primaryKeyComboBox.currentText())
        dc.set_safe(section, 'dedupe_on',             self.dedupeOnHandler.get_model_list(left=False))
        dc.set_safe(section, 'gather_fields',         self.gatherFieldsHandler.get_model_list(left=False))
        dc.set_safe(section, 'gather_fields_overwrite', self.gatherFieldsOverWriteCheckBox.isChecked())
        dc.set_safe(section, 'sort_on',               self.sortOnHandler.get_model_list(left=False))
        dc.set_safe(section, 'sort_ascending',        self.sortAscHandler.get_model_list(left=False))
        dc.set_safe(section, 'unique_fields',         self.uniqueFieldsHandler.get_model_list(left=False))
        dc.set_safe(section, 'field_map_data',        self._field_map_data)
        dc.set_safe(section, 'merge_files',           list(self._merge_files.keys()))
        dc.set_safe(section, 'purge_files',           list(self._purge_files.keys()))
        return dc

    def set_settings(self, dc:DictConfig, section="MERGE_PURGE"):
        """
        Applies settings from a DictConfig object to the Dialog.

        :param dc (DictConfig, default None)
            The DictConfig object that contains the settings to be applied.
        :param section (str, default 'MERGE_PURGE')
            The section name to read settings from.
        :return:
        """
        source_path = dc.get(section, 'source_path', fallback=self.sourcePathLineEdit.text())
        current_path = self.sourcePathLineEdit.text()
        if source_path != current_path:
            dfm = self.df_manager.read_file(source_path)
            dest = dc.get(section, 'dest_path', fallback=None)
            self.set_source_model(dfm, configure=False)
            self.set_line_edit_paths(source_path, dest_path=dest)
            self.primaryKeyComboBox.clear()
            self.set_primary_key_combo_box()

        key_id = self.primaryKeyComboBox.findText(dc.get(section, 'primary_key',
                                                         fallback=self.primaryKeyComboBox.currentText()))

        dedupe_on     = dc.get_safe(section, 'dedupe_on', fallback=None)
        sort_on       = dc.get_safe(section, 'sort_on', fallback=None)
        gather_fields = dc.get_safe(section, 'gather_fields', fallback=None)
        unique_fields = dc.get_safe(section, 'unique_fields', fallback=None)
        gather_fields_overwrite = dc.getboolean(section, 'gather_fields_overwrite', fallback=False)
        sort_ascending = dc.get_safe(section, 'sort_ascending', fallback=None)
        merge_files    = dc.get_safe(section, 'merge_files', fallback=[])
        purge_files    = dc.get_safe(section, 'purge_files', fallback=[])
        field_map_data = dc.get_safe(section, 'field_map_data', fallback={})

        self.primaryKeyComboBox.setCurrentIndex(key_id)
        self.set_push_grid_handlers(column_model=None, sorton_model=sort_on, sortasc_model=sort_ascending,
                               dedupe_model=dedupe_on, gather_model=gather_fields, unique_model=unique_fields)
        self.gatherFieldsOverWriteCheckBox.setChecked(gather_fields_overwrite)
        self._field_map_data.update(field_map_data)
        self.open_file(file_names=merge_files, model_signal=self.signalMergeFileOpened)
        self.open_file(file_names=purge_files, model_signal=self.signalSFileOpened)

    def reset(self):
        """
        Resets ListViews/CheckBoxes.
        The source/dest line edits are left alone
        The suppression/merge files are also left alone.
        :return: None
        """
        self.set_push_grid_handlers()
        self.set_handler_sort_asc(overwrite=True)
        self.set_primary_key_combo_box(reset=True)
        self.gatherFieldsOverWriteCheckBox.setChecked(False)

    def set_primary_key_combo_box(self, default=None, reset=False):
        """
        Sets the primary key combo box.
        :param default: (str, default None)
            An optional default field name to select.
        :return:
        """
        if default is None and reset is False:
            current_model = self.primaryKeyComboBox.model()
            if current_model:
                default = self.primaryKeyComboBox.currentText()

        combo_model = create_standard_item_model([''] + self.source_model.dataFrame().columns.tolist(),
                                                 editable=False, checkable=True)
        self.primaryKeyComboBox.setModel(combo_model)
        if default is not None:
            self.primaryKeyComboBox.setCurrentIndex(self.primaryKeyComboBox.findText(default))

    def import_settings(self, from_path=None):
        """
        Imports settings to the Dialog from a file.
        :param from_path: (str, default None)
            None makes the user enter a file path.
        :return:
        """
        if from_path is None:
            try:
                dirname = os.path.dirname(self.sourcePathLineEdit.text())
            except:
                dirname = ''
            from_path = QtGui.QFileDialog.getOpenFileName(self, dir=dirname)[0]
        config = SettingsINI(filename=from_path)
        self.set_settings(config)

    def export_settings(self, to=None):
        """
        Exports settings from the Dialog to a file.
        :param to: (str, default None)
            None makes the user enter a file path.
        :return: None
        """
        if to is None:
            try:
                dirname = os.path.dirname(self.sourcePathLineEdit.text())
            except:
                dirname = ''
            to = QtGui.QFileDialog.getSaveFileName(self, dir=dirname)[0]
        config = self.get_settings()
        config.save_as(to, set_self=True)


# -*- coding: utf-8 -*- {{{
# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et:

# Copyright (c) 2016, Battelle Memorial Institute
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation
# are those of the authors and should not be interpreted as representing
# official policies, either expressed or implied, of the FreeBSD
# Project.
#
# This material was prepared as an account of work sponsored by an
# agency of the United States Government.  Neither the United States
# Government nor the United States Department of Energy, nor Battelle,
# nor any of their employees, nor any jurisdiction or organization that
# has cooperated in the development of these materials, makes any
# warranty, express or implied, or assumes any legal liability or
# responsibility for the accuracy, completeness, or usefulness or any
# information, apparatus, product, software, or process disclosed, or
# represents that its use would not infringe privately owned rights.
#
# Reference herein to any specific commercial product, process, or
# service by trade name, trademark, manufacturer, or otherwise does not
# necessarily constitute or imply its endorsement, recommendation, or
# favoring by the United States Government or any agency thereof, or
# Battelle Memorial Institute. The views and opinions of authors
# expressed herein do not necessarily state or reflect those of the
# United States Government or any agency thereof.
#
# PACIFIC NORTHWEST NATIONAL LABORATORY
# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY
# under Contract DE-AC05-76RL01830

# }}}
import os
import sys
import logging
import datetime
from dateutil import parser

from volttron.platform.vip.agent import Agent, Core, PubSub, RPC, compat
from volttron.platform.agent import utils
from volttron.platform.agent.utils import (get_aware_utc_now,
                                           format_timestamp)

import pandas as pd
import statsmodels.formula.api as sm

utils.setup_logging()
_log = logging.getLogger(__name__)

class PGnEAgent(Agent):
    def __init__(self, config_path, **kwargs):
        super(PGnEAgent, self).__init__(**kwargs)
        self.config = utils.load_config(config_path)
        self.site = self.config.get('campus')
        self.building = self.config.get('building')
        self.temp_unit = self.config.get('temp_unit')
        self.power_unit = self.config.get('power_unit')
        self.out_temp_name = self.config.get('out_temp_name')
        self.power_name = self.config.get('power_name')
        self.aggregate_in_min = self.config.get('aggregate_in_min')
        self.aggregate_freq = str(self.aggregate_in_min) + 'Min'
        self.ts_name = self.config.get('ts_name')

        self.window_size_in_day = int(self.config.get('window_size_in_day'))
        self.min_required_window_size_in_percent = float(self.config.get('min_required_window_size_in_percent'))
        self.interval_in_min = int(self.config.get('interval_in_min'))
        self.no_of_recs_needed = 10 # self.window_size_in_day * 24 * (60 / self.interval_in_min)
        self.min_no_of_records_needed_after_aggr = int(self.min_required_window_size_in_percent/100 *
                                            self.no_of_recs_needed/self.aggregate_in_min)
        self.schedule_run_in_sec = int(self.config.get('schedule_run_in_hr')) * 3600

        # Testing
        #self.no_of_recs_needed = 200
        #self.min_no_of_records_needed_after_aggr = self.no_of_recs_needed/self.aggregate_in_min

    @Core.receiver('onstart')
    def onstart(self, sender, **kwargs):
        self.core.periodic(self.schedule_run_in_sec, self.calculate_latest_coeffs)

    def calculate_latest_coeffs(self):
        unit_topic_tmpl = "{campus}/{building}/{unit}/{point}"
        unit_points = [self.power_name]
        df = None

        #Get data
        unit = self.temp_unit
        for point in unit_points:
            if point == self.power_name:
                unit = self.power_unit
            unit_topic = unit_topic_tmpl.format(campus=self.site,
                                                building=self.building,
                                                unit=unit,
                                                point=point)
            result = self.vip.rpc.call('platform.historian',
                                       'query',
                                       topic=unit_topic,
                                       count=self.no_of_recs_needed,
                                       order="LAST_TO_FIRST").get(timeout=10000)
            df2 = pd.DataFrame(result['values'], columns=[self.ts_name, point])
            df2[self.ts_name] = pd.to_datetime(df2[self.ts_name])
            df2 = df2.groupby([pd.TimeGrouper(key=self.ts_name, freq=self.aggregate_freq)]).mean()
            # df2[self.ts_name] = df2[self.ts_name].apply(lambda dt: dt.replace(second=0, microsecond=0))
            df = df2 if df is None else pd.merge(df, df2, how='outer', left_index=True, right_index=True)

        #Calculate coefficients
        result_df = self.calculate_coeffs(df)

        # Publish coeffs to store
        #if coeffs is not None:
        #    self.save_coeffs(coeffs, subdevice)

    def convert_units_to_SI(self, df, point, unit):
        if unit == 'degreesFahrenheit':
            df[point] = (df[point]-32) * 5/9
        # Air state assumption: http://www.remak.eu/en/mass-air-flow-rate-unit-converter
        # 1cfm ~ 0.00055kg/s
        if unit == 'cubicFeetPerMinute':
            df[point] = df[point] * 0.00055

    def calculate_coeffs(self, dP):
        dP['time'] = dP['posttime']
        dP = dP.set_index(['posttime'])

        dP.index = pd.to_datetime(dP.index)
        dP['time'] = pd.to_datetime(dP['time'])

        #### Delete the weekend
        dP.columns = ["Tout", "wbe", "Weekday", "time"]
        dP['year'] = dP.index.year
        dP['month'] = dP.index.month
        dP['hour'] = dP.index.hour
        dP['day'] = dP.index.day
        dP = dP[dP.Weekday != 'Sun']
        dP = dP[dP.Weekday != 'Sat']

        ####  Hourly average value
        df = dP.resample('60min').mean()
        dP2 = dP.resample('60min').mean()

        dP = dP[dP.Tout < 150]
        dP = dP[dP.Tout > 20]
        dP = dP.dropna()

        df = df.pivot_table(index=["year", "month", "day"], columns=["hour"], values=["wbe", "Tout"])

        # ### Average using high five outdoor temperature data based on 10 day moving windows

        leng = len(df.index)
        for i in range(0, leng):
            for j in range(0, 24):
                df['power', j] = df.ix[i:i + 10, :].sort([('Tout', j)], ascending=False).head(5).ix[:,
                                 j + 24:j + 25].mean()

        for i in range(0, leng):
            for j in range(0, 24):
                df['power', j][i:i + 1] = df.ix[i:i + 10, :].sort([('Tout', j)], ascending=False).head(5).ix[:,
                                          j + 24:j + 25].mean()

        # ### Average based on 10 day moving windows

        for i in range(0, 24):
            df['Tout_avg', i] = df.ix[:, i:i + 1].rolling(window=10, min_periods=10).mean()

        for i in range(0, 24):
            df['Pow_avg', i] = df.ix[:, i + 24:i + 25].rolling(window=10, min_periods=10).mean()

        df = df.stack(level=['hour'])
        df.power = df.power.shift(216)
        df = df.dropna()
        dq = df.reset_index()
        dq['Data'] = pd.to_datetime(
            dq.year.astype(int).apply(str) + '/' + dq.month.astype(int).apply(str) + '/' + dq.day.astype(int).apply(
                str) + ' ' + dq.hour.astype(int).apply(str) + ":00", format='%Y/%m/%d %H:%M')
        dq = dq.set_index(['Data'])
        dq = dq.drop(['year', 'month', 'day', 'hour'], axis=1)
        dk = dq

        ### Adjusted average using high five outdoor temperature data based on 10 day moving windows
        lengnth = len(dq.index)
        lengnth = lengnth - 4
        dq["Adj"] = 1.0
        for i in range(0, lengnth):
            dq['Adj'][i + 4] = (dq['wbe'][i:i + 4].mean()) / (dq['Pow_avg'][i:i + 4].mean())

        dq['Pow_adj'] = dq['Pow_avg'] * dq['Adj']

        #### Adjusted average based on 10 day moving windows
        lengnth = len(dq.index)
        lengnth = lengnth - 4
        dq["Adj2"] = 1.0
        for i in range(0, lengnth):
            dq['Adj2'][i + 4] = (dq['wbe'][i:i + 4].mean()) / (dq['power'][i:i + 4].mean())

        dq['Adj2'] = dq.Adj2.shift(2)
        dq['power_adj'] = dq['power'] * dq['Adj2']

        return dq

    def save_coeffs(self, coeffs, subdevice):
        topic_tmpl = "analysis/TCM/{campus}/{building}/{unit}/{subdevice}/"
        topic = topic_tmpl.format(campus=self.site,
                                  building=self.building,
                                  unit=self.unit,
                                  subdevice=subdevice)
        T_coeffs = coeffs["T_fit"]
        Q_coeffs = coeffs["Q_fit"]
        headers = {'Date': format_timestamp(get_aware_utc_now())}
        for idx in xrange(0,5):
            T_topic = topic + "T_c" + str(idx)
            Q_topic = topic + "Q_c" + str(idx)
            self.vip.pubsub.publish(
                'pubsub', T_topic, headers, T_coeffs.params[idx])
            self.vip.pubsub.publish(
                'pubsub', Q_topic, headers, Q_coeffs.params[idx])

        _log.debug(T_coeffs.params)
        _log.debug(Q_coeffs.params)

def main(argv=sys.argv):
    '''Main method called by the eggsecutable.'''
    try:
        utils.vip_main(PGnEAgent)
    except Exception as e:
        _log.exception('unhandled exception')

def test_ols():
    '''To compare result of pandas and R's linear regression'''
    import os

    test_csv = '../test_data/tcm_ZONE_VAV_150_data.csv'
    df = pd.read_csv(test_csv)

    config_path = os.environ.get('AGENT_CONFIG')
    tcm = PGnEAgent(config_path)
    coeffs = tcm.calculate_coeffs(df)
    if coeffs is not None:
        T_coeffs = coeffs["T_fit"]
        Q_coeffs = coeffs["Q_fit"]
        _log.debug(T_coeffs.params)
        _log.debug(Q_coeffs.params)

def test_api():
    '''To test Volttron APIs'''
    import os

    topic_tmpl = "{campus}/{building}/{unit}/{subdevice}/{point}"
    tcm = PGnEAgent(os.environ.get('AGENT_CONFIG'))

    topic1 = topic_tmpl.format(campus='PNNL',
                               building='SEB',
                               unit='AHU1',
                               subdevice='VAV123A',
                               point='MaximumZoneAirFlow')
    result = tcm.vip.rpc.call('platform.historian',
                              'query',
                              topic=topic1,
                              count=20,
                              order="LAST_TO_FIRST").get(timeout=100)
    assert result is not None

if __name__ == '__main__':
    # Entry point for script
    sys.exit(main())
    #test_api()