#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Feature manipulation utilities"""

from warnings import warn
import numpy as np
import scipy.signal

from .._cache import cache
from ..util.exceptions import ParameterError
__all__ = ['delta', 'stack_memory']

@cache(level=40)
def delta(data, width=9, order=1, axis=-1, mode='interp', **kwargs):
    r'''Compute delta features: local estimate of the derivative
    of the input data along the selected axis.

    Delta features are computed Savitsky-Golay filtering.

    Parameters
    ----------
    data      : np.ndarray
        the input data matrix (eg, spectrogram)

    width     : int, positive, odd [scalar]
        Number of frames over which to compute the delta features.
        Cannot exceed the length of `data` along the specified axis.
        If `mode='interp'`, then `width` must be at least `data.shape[axis]`.

    order     : int > 0 [scalar]
        the order of the difference operator.
        1 for first derivative, 2 for second, etc.

    axis      : int [scalar]
        the axis along which to compute deltas.
        Default is -1 (columns).

    mode : str, {'interp', 'nearest', 'mirror', 'constant', 'wrap'}
        Padding mode for estimating differences at the boundaries.

    kwargs : additional keyword arguments
        See `scipy.signal.savgol_filter`

    Returns
    -------
    delta_data   : np.ndarray [shape=(d, t)]
        delta matrix of `data` at specified order

    Notes
    -----
    This function caches at level 40.

    See Also
    --------
    scipy.signal.savgol_filter

    Examples
    --------
    Compute MFCC deltas, delta-deltas

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> mfcc = librosa.feature.mfcc(y=y, sr=sr)
    >>> mfcc_delta = librosa.feature.delta(mfcc)
    >>> mfcc_delta
    array([[  1.666e+01,   1.666e+01, ...,   1.869e-15,   1.869e-15],
           [  1.784e+01,   1.784e+01, ...,   6.085e-31,   6.085e-31],
           ...,
           [  7.262e-01,   7.262e-01, ...,   9.259e-31,   9.259e-31],
           [  6.578e-01,   6.578e-01, ...,   7.597e-31,   7.597e-31]])

    >>> mfcc_delta2 = librosa.feature.delta(mfcc, order=2)
    >>> mfcc_delta2
    array([[ -1.703e+01,  -1.703e+01, ...,   3.834e-14,   3.834e-14],
           [ -1.108e+01,  -1.108e+01, ...,  -1.068e-30,  -1.068e-30],
           ...,
           [  4.075e-01,   4.075e-01, ...,  -1.565e-30,  -1.565e-30],
           [  1.676e-01,   1.676e-01, ...,  -2.104e-30,  -2.104e-30]])

    >>> import matplotlib.pyplot as plt
    >>> plt.subplot(3, 1, 1)
    >>> librosa.display.specshow(mfcc)
    >>> plt.title('MFCC')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 2)
    >>> librosa.display.specshow(mfcc_delta)
    >>> plt.title(r'MFCC-$\Delta$')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 3)
    >>> librosa.display.specshow(mfcc_delta2, x_axis='time')
    >>> plt.title(r'MFCC-$\Delta^2$')
    >>> plt.colorbar()
    >>> plt.tight_layout()
    >>> plt.show()

    '''

    data = np.atleast_1d(data)

    if mode == 'interp' and width > data.shape[axis]:
        raise ParameterError("when mode='interp', width={} "
                             "cannot exceed data.shape[axis]={}".format(width, data.shape[axis]))

    if width < 3 or np.mod(width, 2) != 1:
        raise ParameterError('width must be an odd integer >= 3')

    if order <= 0 or not isinstance(order, int):
        raise ParameterError('order must be a positive integer')

    kwargs.pop('deriv', None)
    kwargs.setdefault('polyorder', order)
    return scipy.signal.savgol_filter(data, width,
                                      deriv=order,
                                      axis=axis,
                                      mode=mode,
                                      **kwargs)

@cache(level=40)
def stack_memory(data, n_steps=2, delay=1, **kwargs):
    """Short-term history embedding: vertically concatenate a data
    vector or matrix with delayed copies of itself.

    Each column `data[:, i]` is mapped to::

        data[:, i] ->  [data[:, i],
                        data[:, i - delay],
                        ...
                        data[:, i - (n_steps-1)*delay]]

    For columns `i < (n_steps - 1) * delay` , the data will be padded.
    By default, the data is padded with zeros, but this behavior can be
    overridden by supplying additional keyword arguments which are passed
    to `np.pad()`.

    Parameters
    ----------
    data : np.ndarray [shape=(t,) or (d, t)]
        Input data matrix.  If `data` is a vector (`data.ndim == 1`),
        it will be interpreted as a row matrix and reshaped to `(1, t)`.

    n_steps : int > 0 [scalar]
        embedding dimension, the number of steps back in time to stack

    delay : int != 0 [scalar]
        the number of columns to step.

        Positive values embed from the past (previous columns).

        Negative values embed from the future (subsequent columns).

    kwargs : additional keyword arguments
      Additional arguments to pass to `np.pad`.

    Returns
    -------
    data_history : np.ndarray [shape=(m * d, t)]
        data augmented with lagged copies of itself,
        where `m == n_steps - 1`.

    Notes
    -----
    This function caches at level 40.

    Examples
    --------
    Keep two steps (current and previous)

    >>> data = np.arange(-3, 3)
    >>> librosa.feature.stack_memory(data)
    array([[-3, -2, -1,  0,  1,  2],
           [ 0, -3, -2, -1,  0,  1]])

    Or three steps

    >>> librosa.feature.stack_memory(data, n_steps=3)
    array([[-3, -2, -1,  0,  1,  2],
           [ 0, -3, -2, -1,  0,  1],
           [ 0,  0, -3, -2, -1,  0]])

    Use reflection padding instead of zero-padding

    >>> librosa.feature.stack_memory(data, n_steps=3, mode='reflect')
    array([[-3, -2, -1,  0,  1,  2],
           [-2, -3, -2, -1,  0,  1],
           [-1, -2, -3, -2, -1,  0]])

    Or pad with edge-values, and delay by 2

    >>> librosa.feature.stack_memory(data, n_steps=3, delay=2, mode='edge')
    array([[-3, -2, -1,  0,  1,  2],
           [-3, -3, -3, -2, -1,  0],
           [-3, -3, -3, -3, -3, -2]])

    Stack time-lagged beat-synchronous chroma edge padding

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> chroma = librosa.feature.chroma_stft(y=y, sr=sr)
    >>> tempo, beats = librosa.beat.beat_track(y=y, sr=sr, hop_length=512)
    >>> beats = librosa.util.fix_frames(beats, x_min=0, x_max=chroma.shape[1])
    >>> chroma_sync = librosa.util.sync(chroma, beats)
    >>> chroma_lag = librosa.feature.stack_memory(chroma_sync, n_steps=3,
    ...                                           mode='edge')

    Plot the result

    >>> import matplotlib.pyplot as plt
    >>> beat_times = librosa.frames_to_time(beats, sr=sr, hop_length=512)
    >>> librosa.display.specshow(chroma_lag, y_axis='chroma', x_axis='time',
    ...                          x_coords=beat_times)
    >>> plt.yticks([0, 12, 24], ['Lag=0', 'Lag=1', 'Lag=2'])
    >>> plt.title('Time-lagged chroma')
    >>> plt.colorbar()
    >>> plt.tight_layout()
    >>> plt.show()
    """

    if n_steps < 1:
        raise ParameterError('n_steps must be a positive integer')

    if delay == 0:
        raise ParameterError('delay must be a non-zero integer')

    data = np.atleast_2d(data)

    t = data.shape[1]
    kwargs.setdefault('mode', 'constant')

    if kwargs['mode'] == 'constant':
        kwargs.setdefault('constant_values', [0])

    # Pad the end with zeros, which will roll to the front below
    if delay > 0:
        padding = (int((n_steps - 1) * delay), 0)
    else:
        padding = (0, int((n_steps - 1) * -delay))

    data = np.pad(data, [(0, 0), padding], **kwargs)

    history = data

    # TODO: this could be more efficient
    for i in range(1, n_steps):
        history = np.vstack([np.roll(data, -i * delay, axis=1), history])

    # Trim to original width
    if delay > 0:
        history = history[:, :t]
    else:
        history = history[:, -t:]

    # Make contiguous
    return np.asfortranarray(history)


"""
==================================================
Probability Calibration for 3-class classification
==================================================

This example illustrates how sigmoid calibration changes predicted
probabilities for a 3-class classification problem. Illustrated is the
standard 2-simplex, where the three corners correspond to the three classes.
Arrows point from the probability vectors predicted by an uncalibrated
classifier to the probability vectors predicted by the same classifier after
sigmoid calibration on a hold-out validation set. Colors indicate the true
class of an instance (red: class 1, green: class 2, blue: class 3).

The base classifier is a random forest classifier with 25 base estimators
(trees). If this classifier is trained on all 800 training datapoints, it is
overly confident in its predictions and thus incurs a large log-loss.
Calibrating an identical classifier, which was trained on 600 datapoints, with
method='sigmoid' on the remaining 200 datapoints reduces the confidence of the
predictions, i.e., moves the probability vectors from the edges of the simplex
towards the center. This calibration results in a lower log-loss. Note that an
alternative would have been to increase the number of base estimators which
would have resulted in a similar decrease in log-loss.
"""
print(__doc__)

# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
# License: BSD Style.

import matplotlib.pyplot as plt

import numpy as np

from sklearn.datasets import make_blobs
from sklearn.ensemble import RandomForestClassifier
from sklearn.calibration import CalibratedClassifierCV
from sklearn.metrics import log_loss

np.random.seed(0)

# Generate data
X, y = make_blobs(n_samples=1000, n_features=2, random_state=42,
                  cluster_std=5.0)
X_train, y_train = X[:600], y[:600]
X_valid, y_valid = X[600:800], y[600:800]
X_train_valid, y_train_valid = X[:800], y[:800]
X_test, y_test = X[800:], y[800:]

# Train uncalibrated random forest classifier on whole train and validation
# data and evaluate on test data
clf = RandomForestClassifier(n_estimators=25)
clf.fit(X_train_valid, y_train_valid)
clf_probs = clf.predict_proba(X_test)
score = log_loss(y_test, clf_probs)

# Train random forest classifier, calibrate on validation data and evaluate
# on test data
clf = RandomForestClassifier(n_estimators=25)
clf.fit(X_train, y_train)
clf_probs = clf.predict_proba(X_test)
sig_clf = CalibratedClassifierCV(clf, method="sigmoid", cv="prefit")
sig_clf.fit(X_valid, y_valid)
sig_clf_probs = sig_clf.predict_proba(X_test)
sig_score = log_loss(y_test, sig_clf_probs)

# Plot changes in predicted probabilities via arrows
plt.figure(0)
colors = ["r", "g", "b"]
for i in range(clf_probs.shape[0]):
    plt.arrow(clf_probs[i, 0], clf_probs[i, 1],
              sig_clf_probs[i, 0] - clf_probs[i, 0],
              sig_clf_probs[i, 1] - clf_probs[i, 1],
              color=colors[y_test[i]], head_width=1e-2)

# Plot perfect predictions
plt.plot([1.0], [0.0], 'ro', ms=20, label="Class 1")
plt.plot([0.0], [1.0], 'go', ms=20, label="Class 2")
plt.plot([0.0], [0.0], 'bo', ms=20, label="Class 3")

# Plot boundaries of unit simplex
plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label="Simplex")

# Annotate points on the simplex
plt.annotate(r'($\frac{1}{3}$, $\frac{1}{3}$, $\frac{1}{3}$)',
             xy=(1.0/3, 1.0/3), xytext=(1.0/3, .23), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.plot([1.0/3], [1.0/3], 'ko', ms=5)
plt.annotate(r'($\frac{1}{2}$, $0$, $\frac{1}{2}$)',
             xy=(.5, .0), xytext=(.5, .1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($0$, $\frac{1}{2}$, $\frac{1}{2}$)',
             xy=(.0, .5), xytext=(.1, .5), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($\frac{1}{2}$, $\frac{1}{2}$, $0$)',
             xy=(.5, .5), xytext=(.6, .6), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($0$, $0$, $1$)',
             xy=(0, 0), xytext=(.1, .1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($1$, $0$, $0$)',
             xy=(1, 0), xytext=(1, .1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
plt.annotate(r'($0$, $1$, $0$)',
             xy=(0, 1), xytext=(.1, 1), xycoords='data',
             arrowprops=dict(facecolor='black', shrink=0.05),
             horizontalalignment='center', verticalalignment='center')
# Add grid
plt.grid("off")
for x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
    plt.plot([0, x], [x, 0], 'k', alpha=0.2)
    plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)
    plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)

plt.title("Change of predicted probabilities after sigmoid calibration")
plt.xlabel("Probability class 1")
plt.ylabel("Probability class 2")
plt.xlim(-0.05, 1.05)
plt.ylim(-0.05, 1.05)
plt.legend(loc="best")

print("Log-loss of")
print(" * uncalibrated classifier trained on 800 datapoints: %.3f "
      % score)
print(" * classifier trained on 600 datapoints and calibrated on "
      "200 datapoint: %.3f" % sig_score)

# Illustrate calibrator
plt.figure(1)
# generate grid over 2-simplex
p1d = np.linspace(0, 1, 20)
p0, p1 = np.meshgrid(p1d, p1d)
p2 = 1 - p0 - p1
p = np.c_[p0.ravel(), p1.ravel(), p2.ravel()]
p = p[p[:, 2] >= 0]

calibrated_classifier = sig_clf.calibrated_classifiers_[0]
prediction = np.vstack([calibrator.predict(this_p)
                        for calibrator, this_p in
                        zip(calibrated_classifier.calibrators_, p.T)]).T
prediction /= prediction.sum(axis=1)[:, None]

# Ploit modifications of calibrator
for i in range(prediction.shape[0]):
    plt.arrow(p[i, 0], p[i, 1],
              prediction[i, 0] - p[i, 0], prediction[i, 1] - p[i, 1],
              head_width=1e-2, color=colors[np.argmax(p[i])])
# Plot boundaries of unit simplex
plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label="Simplex")

plt.grid("off")
for x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
    plt.plot([0, x], [x, 0], 'k', alpha=0.2)
    plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)
    plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)

plt.title("Illustration of sigmoid calibrator")
plt.xlabel("Probability class 1")
plt.ylabel("Probability class 2")
plt.xlim(-0.05, 1.05)
plt.ylim(-0.05, 1.05)

plt.show()


from L500analysis.data_io.get_cluster_data import GetClusterData
from L500analysis.utils.utils import aexp2redshift
from L500analysis.plotting.tools.figure_formatting import *
from L500analysis.plotting.profiles.tools.profiles_percentile \
    import *
from L500analysis.plotting.profiles.tools.select_profiles \
    import nu_cut, prune_dict
from L500analysis.utils.constants import rbins
from derived_field_functions import *

color = matplotlib.cm.afmhot_r
aexps = [1.0,0.9,0.8,0.7,0.6,0.5,0.45,0.4,0.35]
nu_threshold = [2.3,2.7]
nu_label = r"%0.1f$\leq\nu_{500c}\leq$%0.1f"%(nu_threshold[0],nu_threshold[1])
db_name = 'L500_NR_0'
db_dir = '/home/babyostrich/Documents/Repos/L500analysis/'

profiles_list = ['r_mid',
                 'Vcirc2_Vc500c',
                 'M_dark', 'M_star', 'M_gas',
                 'R/R500c']

halo_properties_list=['r500c','M_total_500c','nu_500c']

Vcirc2ratioVc500c=r"$\tilde{V}=V^2_{c}/V^2_{c,500c}$"
fVcz1=r"$\tilde{V}/\tilde{V}(z=1)$"

pa = PlotAxes(figname='Vcirc2_Vc500c_nu%0.1f'%nu_threshold[0],
              axes=[[0.15,0.4,0.80,0.55],[0.15,0.15,0.80,0.24]],
              axes_labels=[Vcirc2ratioVc500c,fVcz1],
              xlabel=r"$R/R_{500c}$",
              xlim=(0.2,5),
              ylims=[(0.6,1.4),(0.6,1.4)])

Vcirc2={}
clkeys = ['Vcirc2_Vc500c']
plots = [Vcirc2]
linestyles = ['-']

for aexp in aexps :
    cldata = GetClusterData(aexp=aexp,db_name=db_name,
                            db_dir=db_dir,
                            profiles_list=profiles_list,
                            halo_properties_list=halo_properties_list)

    nu_cut_hids = nu_cut(nu=cldata['nu_500c'], threshold=nu_threshold)

    for plot, key in zip(plots,clkeys) :
        pruned_profiles = prune_dict(d=cldata[key],k=nu_cut_hids)
        plot[aexp] = calculate_profiles_mean_variance(pruned_profiles)

    pa.axes[Vcirc2ratioVc500c].plot( rbins, Vcirc2[aexp]['mean'],color=color(aexp),
                                     ls='-',label="$z=%3.1f$" % aexp2redshift(aexp))

pa.axes[Vcirc2ratioVc500c].fill_between(rbins, Vcirc2[0.5]['down'], Vcirc2[0.5]['up'],
                                 color=color(0.5), zorder=0)

for aexp in aexps :
    for V,ls in zip(plots,linestyles) :
        fractional_evolution = get_profiles_division_mean_variance(
            mean_profile1=V[aexp]['mean'],
            var_profile1=V[aexp]['var'],
            mean_profile2=V[0.5]['mean'],
            var_profile2=V[0.5]['var'],
            )

        pa.axes[fVcz1].plot( rbins, fractional_evolution['mean'],
                             color=color(aexp),ls=ls)

pa.axes[Vcirc2ratioVc500c].annotate(nu_label, xy=(.75, .75), xytext=(.3, 1.3))
pa.axes[Vcirc2ratioVc500c].tick_params(labelsize=12)
pa.axes[Vcirc2ratioVc500c].tick_params(labelsize=12)
pa.axes[fVcz1].set_yticks(arange(0.6,1.4,0.2))

matplotlib.rcParams['legend.handlelength'] = 0
matplotlib.rcParams['legend.numpoints'] = 1
matplotlib.rcParams['legend.fontsize'] = 12
pa.set_legend(axes_label=Vcirc2ratioVc500c,ncol=3,loc='upper right', frameon=False)
pa.color_legend_texts(axes_label=Vcirc2ratioVc500c)

pa.savefig()


# ===================================
# CALCULATES Ioff and Ires
# Indicators described in Molecfit II
#
# Solene 20.09.2016
# ===================================
#
import numpy as np
from astropy.io import fits
import matplotlib.pyplot as plt
# from PyAstronomy import pyasl
from scipy.interpolate import interp1d
from scipy.interpolate import InterpolatedUnivariateSpline
from scipy import stats
# from sklearn.metrics import mean_squared_error
# from math import sqrt
# from numpy import linalg as LA

# MOLECFIT
#
file_molecfit = '/home/solene/atmos/For_Solene/1203nm/output/molecfit_crires_solene_tac.fits'
hdu_molecfit = fits.open(file_molecfit)
data_molecfit = hdu_molecfit[1].data
cols_molecfit = hdu_molecfit[1].columns
# cols_molecfit.info()
rawwl_molecfit = data_molecfit.field('mlambda')
wl_molecfit = rawwl_molecfit*10e2
trans_molecfit = data_molecfit.field('mtrans')
cflux_molecfit = data_molecfit.field('cflux')

# TELFIT
#
file_telfit = '/home/solene/atmos/trans_telfit.txt'
wl_telfit, trans_telfit, wl_datatelfit, flux_datatelfit = np.loadtxt(
    file_telfit, unpack=True)

# Interpolation
f_molecfit = interp1d(wl_molecfit, cflux_molecfit,  kind='cubic')
ftrans_molecfit = interp1d(wl_molecfit, trans_molecfit,  kind='cubic')
# f_tapas = interp1d(wlcorr_tapas, trans_tapas)

# **1** BINNED DATA
#  3 delta-lambda = 0.036
# Mean and std deviation of bins on the telluric CORRECTED spectrum
fluxmean_bin_means, bin_edges, binnumber = stats.binned_statistic(
    wl_datatelfit, f_molecfit(wl_datatelfit), statistic='mean',
    bins=np.floor((wl_datatelfit[-1]-wl_datatelfit[0])/0.036))

fluxstd_bin_means, _, _ = stats.binned_statistic(
    wl_datatelfit, f_molecfit(wl_datatelfit), statistic=np.std,
    bins=np.floor((wl_datatelfit[-1]-wl_datatelfit[0])/0.036))

bin_width = (bin_edges[1] - bin_edges[0])
bin_centers = bin_edges[1:] - bin_width/2

# **2** Bins where average TRANSMISSION is > 0.99
flux_trans_mean_bin_means, _, _ = stats.binned_statistic(
    wl_datatelfit, ftrans_molecfit(wl_datatelfit), statistic='mean',
    bins=np.floor((wl_datatelfit[-1]-wl_datatelfit[0])/0.036))
# cont_bin_means = flux_trans_mean_bin_means[flux_trans_mean_bin_means > 0.99]
ind_cont = np.where(flux_trans_mean_bin_means > 0.99)
ind_out = np.where((flux_trans_mean_bin_means < 0.95) &
                   (flux_trans_mean_bin_means > 0.1))

# plt.plot(bin_centers[ind_cont], flux_trans_mean_bin_means[ind_cont], 'kx')

# **3** Interpolation of the continuum cubic
# f_cont = interp1d(bin_centers[ind_cont], flux_trans_mean_bin_means[ind_cont],  kind='cubic')
# Extrapolation with constant value spline
f_cont = InterpolatedUnivariateSpline(
    bin_centers[ind_cont], flux_trans_mean_bin_means[ind_cont], ext=3)
# bbox=[bin_centers[ind_cont][0], bin_centers[ind_cont][-1]],

# **5** Subtract cont to mean flux
# and Divide offset and std by interpolated continuum mean value
sys_offset = (fluxmean_bin_means - f_cont(bin_centers)) / f_cont(bin_centers)
flux_std = fluxstd_bin_means / f_cont(bin_centers)

# **6** independant WL = Divide by average absorption
absorp_molecfit = 1 - flux_trans_mean_bin_means
sys_offset_final = sys_offset / absorp_molecfit
flux_std_final = flux_std / absorp_molecfit

plt.figure(1)
plt.plot(wl_datatelfit, flux_datatelfit, 'b.-', label='Raw data')
# plt.hlines(flux_bin_means, bin_edges[:-1],
#            bin_edges[1:], colors='g', lw=5, label='binned statistic of data')
plt.plot(bin_centers, fluxmean_bin_means, 'rx-', label='Mean binned data')
plt.plot(bin_centers, fluxstd_bin_means, 'kx-', label='Standard deviation binned data')
plt.legend()

plt.figure(2)
plt.plot(wl_datatelfit, flux_datatelfit, 'g.-', label='Data 2nd detector')
plt.plot(wl_molecfit, trans_molecfit, 'r-', label='Molecfit')
plt.plot(wl_datatelfit, f_molecfit(wl_datatelfit),
         'b-', label='Corrected data - Molecfit')
plt.plot(wl_datatelfit, f_cont(wl_datatelfit),
         'k-', label='Interpolated Continuum')
plt.plot(sys_offset_final[ind_out], flux_std_final[ind_out], 'kx')
plt.plot(flux_trans_mean_bin_means[ind_out],
         sys_offset_final[ind_out], 'kx', label='Ioff vs Transmission')
plt.plot(flux_trans_mean_bin_means[ind_out],
         flux_std_final[ind_out], 'r.', label='Ires vs Transmission')
plt.xlabel('Wavelength (nm)')
plt.ylabel('Transmission')
plt.legend(loc=3.)
plt.show()


"""
The :mod:`sklearn.decomposition` module includes matrix decomposition
algorithms, including among others PCA, NMF or ICA. Most of the algorithms of
this module can be regarded as dimensionality reduction techniques.
"""

from ._nmf import NMF, non_negative_factorization
from ._pca import PCA
from ._incremental_pca import IncrementalPCA
from ._kernel_pca import KernelPCA
from ._sparse_pca import SparsePCA, MiniBatchSparsePCA
from ._truncated_svd import TruncatedSVD
from ._fastica import FastICA, fastica
from ._dict_learning import (dict_learning, dict_learning_online,
                             sparse_encode, DictionaryLearning,
                             MiniBatchDictionaryLearning, SparseCoder)
from ._factor_analysis import FactorAnalysis
from ..utils.extmath import randomized_svd
from ._lda import LatentDirichletAllocation

__all__ = ['DictionaryLearning',
           'FastICA',
           'IncrementalPCA',
           'KernelPCA',
           'MiniBatchDictionaryLearning',
           'MiniBatchSparsePCA',
           'NMF',
           'PCA',
           'SparseCoder',
           'SparsePCA',
           'dict_learning',
           'dict_learning_online',
           'fastica',
           'non_negative_factorization',
           'randomized_svd',
           'sparse_encode',
           'FactorAnalysis',
           'TruncatedSVD',
           'LatentDirichletAllocation']


from __future__ import absolute_import

import unittest
from unittest import skipIf
import warnings

try:
    import pandas as pd
    is_pandas = True
except ImportError as e:
    is_pandas = False

from bokeh.models.sources import DataSource, ColumnDataSource, ServerDataSource

class TestColumnDataSourcs(unittest.TestCase):

    def test_basic(self):
        ds = ColumnDataSource()
        self.assertTrue(isinstance(ds, DataSource))

    def test_init_dict_arg(self):
        data = dict(a=[1], b=[2])
        ds = ColumnDataSource(data)
        self.assertEquals(ds.data, data)
        self.assertEquals(set(ds.column_names), set(data.keys()))

    def test_init_dict_data_kwarg(self):
        data = dict(a=[1], b=[2])
        ds = ColumnDataSource(data=data)
        self.assertEquals(ds.data, data)
        self.assertEquals(set(ds.column_names), set(data.keys()))

    @skipIf(not is_pandas, "pandas not installed")
    def test_init_pandas_arg(self):
        data = dict(a=[1, 2], b=[2, 3])
        df = pd.DataFrame(data)
        ds = ColumnDataSource(df)
        self.assertTrue(set(df.columns).issubset(set(ds.column_names)))
        for key in data.keys():
            self.assertEquals(list(df[key]), data[key])
        self.assertEqual(set(ds.column_names) - set(df.columns), set(["index"]))

    @skipIf(not is_pandas, "pandas not installed")
    def test_init_pandas_data_kwarg(self):
        data = dict(a=[1, 2], b=[2, 3])
        df = pd.DataFrame(data)
        ds = ColumnDataSource(data=df)
        self.assertTrue(set(df.columns).issubset(set(ds.column_names)))
        for key in data.keys():
            self.assertEquals(list(df[key]), data[key])
        self.assertEqual(set(ds.column_names) - set(df.columns), set(["index"]))

    def test_add_with_name(self):
        ds = ColumnDataSource()
        name = ds.add([1,2,3], name="foo")
        self.assertEquals(name, "foo")
        name = ds.add([4,5,6], name="bar")
        self.assertEquals(name, "bar")

    def test_add_without_name(self):
        ds = ColumnDataSource()
        name = ds.add([1,2,3])
        self.assertEquals(name, "Series 0")
        name = ds.add([4,5,6])
        self.assertEquals(name, "Series 1")

    def test_add_with_and_without_name(self):
        ds = ColumnDataSource()
        name = ds.add([1,2,3], "foo")
        self.assertEquals(name, "foo")
        name = ds.add([4,5,6])
        self.assertEquals(name, "Series 1")

    def test_remove_exists(self):
        ds = ColumnDataSource()
        name = ds.add([1,2,3], "foo")
        assert name
        ds.remove("foo")
        self.assertEquals(ds.column_names, [])

    def test_remove_exists2(self):
        with warnings.catch_warnings(record=True) as w:
            ds = ColumnDataSource()
            ds.remove("foo")
            self.assertEquals(ds.column_names, [])
            self.assertEquals(len(w), 1)
            self.assertEquals(w[0].category, UserWarning)
            self.assertEquals(str(w[0].message), "Unable to find column 'foo' in data source")

class TestServerDataSources(unittest.TestCase):

    def test_basic(self):
        ds = ServerDataSource()
        self.assertTrue(isinstance(ds, DataSource))

if __name__ == "__main__":
    unittest.main()


"""Implementation of HyperLogLog

This implements the HyperLogLog algorithm for cardinality estimation, found
in

    Philippe Flajolet, Éric Fusy, Olivier Gandouet and Frédéric Meunier.
        "HyperLogLog: the analysis of a near-optimal cardinality estimation
        algorithm". 2007 Conference on Analysis of Algorithms. Nice, France
        (2007)

"""
import numpy as np
import pandas as pd
from pandas.util import hash_pandas_object

def compute_first_bit(a):
    "Compute the position of the first nonzero bit for each int in an array."
    # TODO: consider making this less memory-hungry
    bits = np.bitwise_and.outer(a, 1 << np.arange(32))
    bits = bits.cumsum(axis=1).astype(bool)
    return 33 - bits.sum(axis=1)

def compute_hll_array(obj, b):
    # b is the number of bits

    if not 8 <= b <= 16:
        raise ValueError("b should be between 8 and 16")
    num_bits_discarded = 32 - b
    m = 1 << b

    # Get an array of the hashes
    hashes = hash_pandas_object(obj, index=False)
    if isinstance(hashes, pd.Series):
        hashes = hashes._values
    hashes = hashes.astype(np.uint32)

    # Of the first b bits, which is the first nonzero?
    j = hashes >> num_bits_discarded
    first_bit = compute_first_bit(hashes)

    # Pandas can do the max aggregation
    df = pd.DataFrame({"j": j, "first_bit": first_bit})
    series = df.groupby("j").max()["first_bit"]

    # Return a dense array so we can concat them and get a result
    # that is easy to deal with
    return series.reindex(np.arange(m), fill_value=0).values.astype(np.uint8)

def reduce_state(Ms, b):
    m = 1 << b

    # We concatenated all of the states, now we need to get the max
    # value for each j in both
    Ms = Ms.reshape((len(Ms) // m), m)
    return Ms.max(axis=0)

def estimate_count(Ms, b):
    m = 1 << b

    # Combine one last time
    M = reduce_state(Ms, b)

    # Estimate cardinality, no adjustments
    alpha = 0.7213 / (1 + 1.079 / m)
    E = alpha * m / (2.0 ** -(M.astype("f8"))).sum() * m
    #                        ^^^^ starts as unsigned, need a signed type for
    #                             negation operator to do something useful

    # Apply adjustments for small / big cardinalities, if applicable
    if E < 2.5 * m:
        V = (M == 0).sum()
        if V:
            return m * np.log(m / V)
    if E > 2 ** 32 / 30.0:
        return -(2 ** 32) * np.log1p(-E / 2 ** 32)
    return E


"""
Read a rvcf file with stability selection scores for taxa.
Sort the dataframe by rsq_median.
Print results.

usage:
    python sort_results.py \
        ../example/stability_selection_example_output.vcf \
        ../example/hominid_example_taxon_table_input.txt \
        arcsinsqrt \
        0.5 \
        10
"""
import argparse
import sys

import pandas as pd

from hominid.hominid import read_taxon_file, align_snp_and_taxa

def sort_results(rvcf_input_file_path, taxon_table_file_path, transform,
                 r_sqr_median_cutoff, stability_cutoff, snp_count, no_tables,
                 extra_columns):
    print('plotting {} SNPs from {}'.format(snp_count, rvcf_input_file_path))

    # read the rvcf file and sort by rsq_median
    df = pd.read_csv(rvcf_input_file_path, sep='\t', dtype={'CHROM': str})
    #print('df.shape: {}'.format(df.shape))

    sorted_rsq_best_medians_df = df.sort_values(by='rsq_median', ascending=False)

    x_df = sorted_rsq_best_medians_df[sorted_rsq_best_medians_df.rsq_median > r_sqr_median_cutoff]
    print('{} SNPs with r_sqr > {:5.3f}'.format(x_df.shape[0], r_sqr_median_cutoff))

    taxon_table_df = read_taxon_file(taxon_table_file_path, transform=transform)

    for row_i in range(sorted_rsq_best_medians_df.shape[0]):
        if row_i >= snp_count:
            break
        else:
            # get a 1-row dataframe
            snp_df = sorted_rsq_best_medians_df.iloc[[row_i]]
            aligned_snp_df, aligned_taxa_df = align_snp_and_taxa(
                snp_df,
                taxon_table_df
            )
            # get the taxon stability selection scores
            # use the taxon table df index to get column names for snp_df
            taxon_scores_df = snp_df.loc[:, taxon_table_df.index].transpose()
            sorted_taxon_scores_df = taxon_scores_df.sort_values(by=taxon_scores_df.columns[0], ascending=False)
            #sorted_taxon_scores_df = taxon_scores_df.sort(taxon_scores_df.columns[0], ascending=False)

            p_df_list = []
            print('{} {} {:5.3f}'.format(snp_df.iloc[0].GENE, snp_df.iloc[0].ID, snp_df.iloc[0].rsq_median))
            summary_line = '{}\t{}\t'.format(snp_df.iloc[0].GENE, snp_df.iloc[0].ID)
            for i, (selected_taxon, selected_taxon_row) in enumerate(sorted_taxon_scores_df.iterrows()):
                # use selected_taxon_row.index[0] to index the first and only column
                selected_taxon_score = selected_taxon_row.iloc[0]
                if selected_taxon_score < stability_cutoff:
                    #print('done with selected taxa')
                    break
                else:
                    # trim 'Root;' from the front of the taxon name
                    if selected_taxon.startswith('Root;'):
                        taxon_name = selected_taxon[5:]
                    else:
                        taxon_name = selected_taxon
                    print('  {:5.3f} {}'.format(selected_taxon_score, taxon_name))
                    summary_line += '{}, '.format(taxon_name)
                    gts = [
                        snp_df.iloc[0].REF + snp_df.iloc[0].REF,  # 0
                        snp_df.iloc[0].REF + snp_df.iloc[0].ALT,  # 1
                        snp_df.iloc[0].ALT + snp_df.iloc[0].ALT   # 2
                    ]
                    aligned_snp_value_list = aligned_snp_df.values.flatten().tolist()
                    data_dict = {
                        'chromosome': [snp_df.iloc[0].CHROM] * aligned_snp_df.shape[1],
                        'snp_id': [snp_df.iloc[0].ID] * aligned_snp_df.shape[1],
                        'gene': [snp_df.iloc[0].GENE] * aligned_snp_df.shape[1],
                        'taxon': [selected_taxon] * aligned_snp_df.shape[1],
                        'abundance': aligned_taxa_df[selected_taxon].values.tolist(),
                        'variant_allele_count': [str(int(v)) for v in aligned_snp_value_list],
                        'genotype': [gts[int(v)] for v in aligned_snp_value_list],
                        'sample_id' : aligned_snp_df.columns
                    }
                    columns_to_display = ['abundance', 'variant_allele_count', 'genotype', 'sample_id']
                    if extra_columns:
                        for extra_column in extra_columns.split(','):
                            data_dict[extra_column] = snp_df.iloc[0][extra_column]
                            columns_to_display.append(extra_column)
                    p_df = pd.DataFrame(data_dict)
                    p_df_list.append(p_df)
                    if no_tables:
                        pass
                    else:
                        p_df[columns_to_display].to_csv(
                            sys.stdout,
                            sep='\t'
                        )
            # save a stacked bar plot
            if len(p_df_list) > 0:
                file_name = 'stacked_bar_plot_selected_taxa_{}_{}.pdf'.format(
                    snp_df.iloc[0].GENE,
                    snp_df.iloc[0].ID
                )
                p_df = pd.concat(p_df_list, axis=0)
                # at this point the index for p_df looks like
                #   0...76.0...76.0...76
                # replace the index
                p_df.index = range(p_df.shape[0])
                #p_df.to_csv(file_path, sep='\t')
                stacked_bar_title = '{}\n{}'.format(snp_df.iloc[0].GENE, snp_df.iloc[0].ID)

def main():
    argparser = argparse.ArgumentParser()
    argparser.add_argument('rvcf_input_file_path')
    argparser.add_argument('taxon_table_file_path')
    argparser.add_argument('transform')
    argparser.add_argument(
        'r_sqr_median_cutoff',
        type=float
    )
    argparser.add_argument(
        'stability_cutoff',
        type=float
    )
    argparser.add_argument(
        'snp_count',
        type=int
    )
    argparser.add_argument(
        '--no-tables',
        action='store_true'
    )
    argparser.add_argument(
        '--extra-columns',
        type=str
    )
    args = argparser.parse_args()
    print(args)
    sort_results(**vars(args))

if __name__ == '__main__':
    main()


"""This test for the LFW require medium-size data dowloading and processing

If the data has not been already downloaded by runnning the examples,
the tests won't run (skipped).

If the test are run, the first execution will be long (typically a bit
more than a couple of minutes) but as the dataset loader is leveraging
joblib, successive runs will be fast (less than 200ms).
"""

import random
import os
import shutil
import tempfile
import numpy as np
try:
    try:
        from scipy.misc import imsave
    except ImportError:
        from scipy.misc.pilutil import imsave
except ImportError:
    imsave = None

from sklearn.datasets import load_lfw_pairs
from sklearn.datasets import load_lfw_people

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import raises

SCIKIT_LEARN_DATA = tempfile.mkdtemp(prefix="scikit_learn_lfw_test_")
SCIKIT_LEARN_EMPTY_DATA = tempfile.mkdtemp(prefix="scikit_learn_empty_test_")

LFW_HOME = os.path.join(SCIKIT_LEARN_DATA, 'lfw_home')
FAKE_NAMES = [
    'Abdelatif_Smith',
    'Abhati_Kepler',
    'Camara_Alvaro',
    'Chen_Dupont',
    'John_Lee',
    'Lin_Bauman',
    'Onur_Lopez',
]

def setup_module():
    """Test fixture run once and common to all tests of this module"""
    if imsave is None:
        raise SkipTest

    if not os.path.exists(LFW_HOME):
        os.makedirs(LFW_HOME)

    random_state = random.Random(42)
    np_rng = np.random.RandomState(42)

    # generate some random jpeg files for each person
    counts = {}
    for name in FAKE_NAMES:
        folder_name = os.path.join(LFW_HOME, 'lfw_funneled', name)
        if not os.path.exists(folder_name):
            os.makedirs(folder_name)

        n_faces = np_rng.randint(1, 5)
        counts[name] = n_faces
        for i in range(n_faces):
            file_path = os.path.join(folder_name, name + '_%04d.jpg' % i)
            uniface = np_rng.randint(0, 255, size=(250, 250, 3))
            try:
                imsave(file_path, uniface)
            except ImportError:
                # PIL is not properly installed, skip those tests
                raise SkipTest

    # add some random file pollution to test robustness
    with open(os.path.join(LFW_HOME, 'lfw_funneled', '.test.swp'), 'wb') as f:
        f.write('Text file to be ignored by the dataset loader.')

    # generate some pairing metadata files using the same format as LFW
    with open(os.path.join(LFW_HOME, 'pairsDevTrain.txt'), 'wb') as f:
        f.write("10\n")
        more_than_two = [name for name, count in counts.iteritems()
                         if count >= 2]
        for i in range(5):
            name = random_state.choice(more_than_two)
            first, second = random_state.sample(range(counts[name]), 2)
            f.write('%s\t%d\t%d\n' % (name, first, second))

        for i in range(5):
            first_name, second_name = random_state.sample(FAKE_NAMES, 2)
            first_index = random_state.choice(range(counts[first_name]))
            second_index = random_state.choice(range(counts[second_name]))
            f.write('%s\t%d\t%s\t%d\n' % (first_name, first_index,
                                          second_name, second_index))

    with open(os.path.join(LFW_HOME, 'pairsDevTest.txt'), 'wb') as f:
        f.write("Fake place holder that won't be tested")

    with open(os.path.join(LFW_HOME, 'pairs.txt'), 'wb') as f:
        f.write("Fake place holder that won't be tested")

def teardown_module():
    """Test fixture (clean up) run once after all tests of this module"""
    if os.path.isdir(SCIKIT_LEARN_DATA):
        shutil.rmtree(SCIKIT_LEARN_DATA)
    if os.path.isdir(SCIKIT_LEARN_EMPTY_DATA):
        shutil.rmtree(SCIKIT_LEARN_EMPTY_DATA)

@raises(IOError)
def test_load_empty_lfw_people():
    load_lfw_people(data_home=SCIKIT_LEARN_EMPTY_DATA)

def test_load_fake_lfw_people():
    lfw_people = load_lfw_people(data_home=SCIKIT_LEARN_DATA,
                                 min_faces_per_person=3)

    # The data is croped around the center as a rectangular bounding box
    # arounthe the face. Colors are converted to gray levels:
    assert_equal(lfw_people.images.shape, (10, 62, 47))
    assert_equal(lfw_people.data.shape, (10, 2914))

    # the target is array of person integer ids
    assert_array_equal(lfw_people.target, [2, 0, 1, 0, 2, 0, 2, 1, 1, 2])

    # names of the persons can be found using the target_names array
    expected_classes = ['Abdelatif Smith', 'Abhati Kepler', 'Onur Lopez']
    assert_array_equal(lfw_people.target_names, expected_classes)

    # It is possible to ask for the original data without any croping or color
    # conversion and not limit on the number of picture per person
    lfw_people = load_lfw_people(data_home=SCIKIT_LEARN_DATA,
                                 resize=None, slice_=None, color=True)
    assert_equal(lfw_people.images.shape, (17, 250, 250, 3))

    # the ids and class names are the same as previously
    assert_array_equal(lfw_people.target,
                       [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2])
    assert_array_equal(lfw_people.target_names,
                      ['Abdelatif Smith', 'Abhati Kepler', 'Camara Alvaro',
                       'Chen Dupont', 'John Lee', 'Lin Bauman', 'Onur Lopez'])

@raises(ValueError)
def test_load_fake_lfw_people_too_restrictive():
    load_lfw_people(data_home=SCIKIT_LEARN_DATA, min_faces_per_person=100)

@raises(IOError)
def test_load_empty_lfw_pairs():
    load_lfw_pairs(data_home=SCIKIT_LEARN_EMPTY_DATA)

def test_load_fake_lfw_pairs():
    lfw_pairs_train = load_lfw_pairs(data_home=SCIKIT_LEARN_DATA)

    # The data is croped around the center as a rectangular bounding box
    # arounthe the face. Colors are converted to gray levels:
    assert_equal(lfw_pairs_train.pairs.shape, (10, 2, 62, 47))

    # the target is whether the person is the same or not
    assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])

    # names of the persons can be found using the target_names array
    expected_classes = ['Different persons', 'Same person']
    assert_array_equal(lfw_pairs_train.target_names, expected_classes)

    # It is possible to ask for the original data without any croping or color
    # conversion
    lfw_pairs_train = load_lfw_pairs(data_home=SCIKIT_LEARN_DATA,
                                     resize=None, slice_=None, color=True)
    assert_equal(lfw_pairs_train.pairs.shape, (10, 2, 250, 250, 3))

    # the ids and class names are the same as previously
    assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
    assert_array_equal(lfw_pairs_train.target_names, expected_classes)


# -*- coding: utf-8 -*-
# Copyright 2014-2016 The HyperSpyUI developers
#
# This file is part of HyperSpyUI.
#
# HyperSpyUI is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# HyperSpyUI is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with HyperSpyUI.  If not, see <http://www.gnu.org/licenses/>.
"""
Created on Fri Dec 12 23:44:01 2014

@author: Vidar Tonaas Fauske
"""

from hyperspyui.plugins.plugin import Plugin

from qtpy import QtCore, QtWidgets
from qtpy.QtWidgets import QDialog, QDialogButtonBox, QLineEdit, QLabel

from hyperspy.learn.mva import LearningResults
from hyperspyui.util import win2sig, fig2win, Namespace
from hyperspyui.threaded import ProgressThreaded, ProcessCanceled
from hyperspyui.widgets.extendedqwidgets import ExToolWindow

def tr(text):
    return QtCore.QCoreApplication.translate("MVA", text)

def align_yaxis(ax1, v1, ax2, v2):
    """adjust ax2 ylimit so that v2 in ax2 is aligned to v1 in ax1"""
    _, y1 = ax1.transData.transform((0, v1))
    _, y2 = ax2.transData.transform((0, v2))
    if y2 > y1:
        ratio = y1 / y2
    else:
        ratio = y2 / y1
    inv = ax2.transData.inverted()
    _, dy = inv.transform((0, 0)) - inv.transform((0, y1 - y2))
    miny2, maxy2 = ax2.get_ylim()
    ax2.set_ylim((miny2 + dy) / ratio, (maxy2 + dy) / ratio)

def make_advanced_dialog(ui, algorithms=None):
    diag = ExToolWindow(ui)

    diag.setWindowTitle("Decomposition parameters")

    vbox = QtWidgets.QVBoxLayout()
    if algorithms:
        lbl_algo = QLabel(tr("Choose algorithm:"))
        cbo_algo = QLineEdit.QComboBox()
        cbo_algo.addItems(algorithms)

        vbox.addWidget(lbl_algo)
        vbox.addWidget(cbo_algo)
    else:
        lbl_comp = QLabel(tr(
            "Enter a comma-separated list of component numbers to use for "
            "the model:"))
        txt_comp = QLineEdit()
        vbox.addWidget(lbl_comp)
        vbox.addWidget(txt_comp)

    btns = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel,
                            QtCore.Qt.Horizontal)
    btns.accepted.connect(diag.accept)
    btns.rejected.connect(diag.reject)
    vbox.addWidget(btns)

    diag.setLayout(vbox)

    diag.algorithm = lambda: cbo_algo.currentText()
    diag.components = lambda: [int(s) for s in txt_comp.text().split(',')]
    return diag

class MVA_Plugin(Plugin):

    """
    Implements MVA decomposition utilities.
    """
    name = 'MVA'    # Used for settings groups etc

    coc_values = {'convert': tr("Convert"),
                  'copy': tr("Copy")}

    # ----------- Plugin interface -----------
    def create_actions(self):
        self.settings.set_default('convert_or_copy', None)
        self.settings.set_enum_hint('convert_or_copy',
                                    self.coc_values.keys())
        self.add_action('plot_decomposition_results',
                        tr("Decompose"),
                        self.plot_decomposition_results,
                        icon='pca.svg',
                        tip=tr("Decompose signal using Principle Component "
                               "analysis"),
                        selection_callback=self.selection_rules)
        self.add_action('pca', tr("Decomposition model"), self.pca,
                        icon='pca.svg',
                        tip=tr("Create a Principal Component Analysis "
                               "decomposition model"),
                        selection_callback=self.selection_rules)
        self.add_action('bss', tr("BSS"), self.bss,
                        icon='bss.svg',
                        tip=tr("Run Blind Source Separation"),
                        selection_callback=self.selection_rules)
        self.add_action('bss_model', tr("BSS model"), self.bss_model,
                        icon='bss.svg',
                        tip=tr("Create a Blind Source Separation "
                               "decomposition model"),
                        selection_callback=self.selection_rules)
        self.add_action('clear', tr("Clear"), self.clear,
                        tip=tr("Clear decomposition cache"),
                        selection_callback=self.selection_rules)

    def create_menu(self):
        self.add_menuitem('Decomposition',
                          self.ui.actions['plot_decomposition_results'])
        self.add_menuitem('Decomposition', self.ui.actions['pca'])
        self.add_menuitem('Decomposition', self.ui.actions['bss'])
        self.add_menuitem('Decomposition', self.ui.actions['bss_model'])
        self.add_menuitem('Decomposition', self.ui.actions['clear'])

    def create_toolbars(self):
        self.add_toolbar_button("Decomposition", self.ui.actions['pca'])
        self.add_toolbar_button("Decomposition", self.ui.actions['bss'])

    def selection_rules(self, win, action):
        """
        Callback to determine if action is valid for the passed window.
        """
        s = win2sig(win, self.ui.signals)
        if s is None or s.signal.data.ndim <= 1:
            action.setEnabled(False)
        else:
            action.setEnabled(True)

    # ------------ Action implementations --------------

    def _get_signal(self, signal):
        """
        Get a valid signal. If the signal is none, it ues the currently
        selected one. If the signal type is not float, it either converts it,
        or gets a copy of the correct type, depending on the 'convert_copy'
        setting.
        """
        if signal is None:
            signal = self.ui.get_selected_wrapper()
        s = signal.signal

        if s.data.dtype.char not in ['e', 'f', 'd']:  # If not float
            cc = self.settings.get_or_prompt(
                'convert_or_copy',
                [kv for kv in self.coc_values.items()],
                title=tr("Convert or copy"),
                descr=tr(
                    "Signal data has the wrong data type (float needed)." +
                    "Would you like to convert the current signal, or " +
                    "perform the decomposition on a copy?"))
            if cc is None:
                # User canceled
                raise ProcessCanceled()
            if cc == 'copy':
                s = s.deepcopy()
                s.metadata.General.title = signal.name + "[float]"
                s.plot()
            s.change_dtype(float)
        return s, signal

    def _do_decomposition(self, s, force=False, algorithm=None):
        """
        Makes sure we have decomposition results. If results already are
        available, it will only recalculate if the `force` parameter is True.
        """
        if algorithm:
            s.decomposition(algorithm=algorithm)
        elif force or s.learning_results.explained_variance_ratio is None:
            s.decomposition()
        return s

    def _do_bss(self, s, n_components, algorithm=None):
        """
        Makes sure we have BSS results. If results already are available, it
        will only recalculate if the `force` parameter is True.
        """
        if algorithm:
            s.blind_source_separation(n_components, algorithm=algorithm)
        else:
            s.blind_source_separation(n_components)

    def get_bss_results(self, signal):
        factors = signal.get_bss_factors()
        loadings = signal.get_bss_loadings()
        factors.axes_manager._axes[0] = loadings.axes_manager._axes[0]
        return loadings, factors

    def _record(self, autosig, model, signal, n_components):
        if autosig:
            self.record_code(r"<p>.{0}(n_components={1})".format(
                             model, n_components))
        else:
            self.record_code(r"<p>.{0}({1}, n_components={2})".format(
                             model, signal, n_components))

    def _decompose_threaded(self, callback, label, signal=None,
                            algorithm=None, ns=None):
        if ns is None:
            ns = Namespace()
            ns.autosig = signal is None
            ns.s, signal = self._get_signal(signal)

        def do_threaded():
            ns.s = self._do_decomposition(ns.s, algorithm=algorithm)

        def on_error(message=None):
            em = QtWidgets.QErrorMessage(self.ui)
            msg = tr("An error occurred during decomposition")
            if message:
                msg += ":\n" + message
            em.setWindowTitle(tr("Decomposition error"))
            em.showMessage(msg)

        t = ProgressThreaded(self.ui, do_threaded, lambda: callback(ns),
                             label=label)
        t.worker.error[str].connect(on_error)
        t.run()

    def _perform_model(self, ns, n_components):
        # Num comp. picked, get model, wrap new signal and plot
        if ns.model == 'pca':
            sc = ns.s.get_decomposition_model(n_components)
            sc.metadata.General.title = ns.signal.name + "[PCA-model]"
            sc.plot()
        elif ns.model == 'bss' or ns.model.startswith('bss.'):
            if ns.model.startswith('bss.'):
                algorithm = ns.model[len('bss.'):]
                self._do_bss(ns.s, n_components, algorithm=algorithm)
            else:
                self._do_bss(ns.s, n_components)
            f, o = self.get_bss_results(ns.s)
            o.metadata.add_dictionary(ns.s.metadata.as_dictionary())
            f.metadata.General.title = ns.signal.name + "[BSS-Factors]"
            o.metadata.General.title = ns.signal.name + "[BSS-Loadings]"
            f.plot()
            o.plot()
        elif ns.model == 'bss_model':
            # Here we have to assume the user has actually performed the BSS
            # decomposition first!
            sc = ns.s.get_bss_model(n_components)
            sc.metadata.General.title = ns.signal.name + "[BSS-model]"
            sc.plot()
        if not ns.recorded:
            self._record(ns.autosig, ns.model, ns.signal, n_components)

    def _show_scree(self, ns, callback):
        ax = ns.s.plot_explained_variance_ratio()

        # Clean up plot and present, allow user to select components
        # by picker
        ax.set_title("")
        scree = ax.get_figure().canvas
        scree.draw()
        scree.setWindowTitle("Pick number of components")

        def clicked(event):
            n_components = int(round(event.xdata))
            # Close scree plot
            w = fig2win(scree.figure, self.ui.figures)
            w.close()
            callback(ns, n_components)
        scree.mpl_connect('button_press_event', clicked)

    def do_after_scree(self, model, signal=None, n_components=None):
        """
        Performs decomposition, then plots the scree for the user to select
        the number of components to use for a decomposition model. The
        selection is made by clicking on the scree, which closes the scree
        and creates the model.
        """
        ns = Namespace()
        ns.autosig = signal is None
        ns.model = model
        ns.s, ns.signal = self._get_signal(signal)
        if n_components is not None:
            self._record(ns.autosig, ns.model, ns.signal, n_components)
            ns.recorded = True
        else:
            ns.recorded = False

        def on_complete(ns):
            if n_components is None:
                self._show_scree(ns, self._perform_model)
            else:
                self._perform_model(ns, n_components)

        self._decompose_threaded(on_complete, "Performing %s" % model.upper(),
                                 n_components, ns=ns)

    def plot_decomposition_results(self, signal=None, advanced=False):
        """
        Performs decomposition if necessary, then plots the decomposition
        results according to the hyperspy implementation.
        """
        def on_complete(ns):
            ns.s.plot_decomposition_results()
            # Somewhat speculative workaround to HSPY not adding metadata
            sd = self.ui.hspy_signals[-1]
            sd.metadata.add_dictionary(ns.s.metadata.as_dictionary())

        if advanced:
            diag = make_advanced_dialog(
                self.ui, ['svd', 'fast_svd', 'mlpca', 'fast_mlpca', 'nmf',
                          'sparse_pca', 'mini_batch_sparse_pca'])
            dr = diag.exec_()
            if dr == QDialog.Accepted:
                self._decompose_threaded(
                    on_complete, "Decomposing signal",
                    algorithm=diag.algorithm())
        else:
            self._decompose_threaded(on_complete, "Decomposing signal")

    def pca(self, signal=None, n_components=None, advanced=False):
        """
        Performs decomposition, then plots the scree for the user to select
        the number of components to use for a decomposition model. The
        selection is made by clicking on the scree, which closes the scree
        and creates the model.
        """
        if advanced:
            diag = make_advanced_dialog(self.ui)
            dr = diag.exec_()
            if dr == QDialog.Accepted:
                self.do_after_scree(
                    'pca', signal, n_components=diag.components())
        else:
            self.do_after_scree('pca', signal, n_components)

    def bss(self, signal=None, n_components=None, advanced=False):
        """
        Performs decomposition if neccessary, then plots the scree for the user
        to select the number of components to use for a blind source
        separation. The selection is made by clicking on the scree, which
        closes the scree and creates the model.
        """
        if advanced:
            diag = make_advanced_dialog(
                self.ui, ['orthomax', 'sklearn_fastica', 'FastICA', 'JADE',
                          'CuBICA', 'TDSEP'])
            dr = diag.exec_()
            if dr == QDialog.Accepted:
                model = 'bss.' + diag.algorithm()
                self.do_after_scree(model, signal, n_components)
        else:
            self.do_after_scree('bss', signal, n_components)

    def bss_model(self, signal=None, n_components=None, advanced=False):
        """
        Performs decomposition if neccessary, then plots the scree for the user
        to select the number of components to use for a blind source
        separation model. The selection is made by clicking on the scree, which
        closes the scree and creates the model.
        """
        if advanced:
            diag = make_advanced_dialog(self.ui)
            dr = diag.exec_()
            if dr == QDialog.Accepted:
                self.do_after_scree(
                    'bss_model', signal, n_components=diag.components())
        else:
            self.do_after_scree('bss_model', signal, n_components)

    def clear(self, signal=None):
        """
        Clears the learning results from the signal.
        """
        if signal is None:
            signal = self.ui.get_selected_signal()
        signal.learning_results = LearningResults()


import numpy as np
import matplotlib.pyplot as plt

d = np.arange(100).reshape(10, 10)
x, y = np.meshgrid(np.arange(11), np.arange(11))

theta = 0.25*np.pi
xx = x*np.cos(theta) - y*np.sin(theta)
yy = x*np.sin(theta) + y*np.cos(theta)

ax1 = plt.subplot(221)
ax1.set_aspect(1)
ax1.pcolormesh(xx, yy, d)
ax1.set_title("No Rasterization")

ax2 = plt.subplot(222)
ax2.set_aspect(1)
ax2.set_title("Rasterization")

m = ax2.pcolormesh(xx, yy, d)
m.set_rasterized(True)

ax3 = plt.subplot(223)
ax3.set_aspect(1)
ax3.pcolormesh(xx, yy, d)
ax3.text(0.5, 0.5, "Text", alpha=0.2,
         va="center", ha="center", size=50, transform=ax3.transAxes)

ax3.set_title("No Rasterization")

ax4 = plt.subplot(224)
ax4.set_aspect(1)
m = ax4.pcolormesh(xx, yy, d)
m.set_zorder(-20)

ax4.text(0.5, 0.5, "Text", alpha=0.2,
         zorder=-15,
         va="center", ha="center", size=50, transform=ax4.transAxes)

ax4.set_rasterization_zorder(-10)

ax4.set_title("Rasterization z$<-10$")

# ax2.title.set_rasterized(True) # should display a warning

plt.savefig("test_rasterization.pdf", dpi=150)
plt.savefig("test_rasterization.eps", dpi=150)

if not plt.rcParams["text.usetex"]:
    plt.savefig("test_rasterization.svg", dpi=150)
    # svg backend currently ignores the dpi


"""
==================================================
Explicit feature map approximation for RBF kernels
==================================================

An example illustrating the approximation of the feature map
of an RBF kernel.

.. currentmodule:: sklearn.kernel_approximation

It shows how to use :class:`RBFSampler` and :class:`Nystroem` to
approximate the feature map of an RBF kernel for classification with an SVM on
the digits dataset. Results using a linear SVM in the original space, a linear
SVM using the approximate mappings and using a kernelized SVM are compared.
Timings and accuracy for varying amounts of Monte Carlo samplings (in the case
of :class:`RBFSampler`, which uses random Fourier features) and different sized
subsets of the training set (for :class:`Nystroem`) for the approximate mapping
are shown.

Please note that the dataset here is not large enough to show the benefits
of kernel approximation, as the exact SVM is still reasonably fast.

Sampling more dimensions clearly leads to better classification results, but
comes at a greater cost. This means there is a tradeoff between runtime and
accuracy, given by the parameter n_components. Note that solving the Linear
SVM and also the approximate kernel SVM could be greatly accelerated by using
stochastic gradient descent via :class:`sklearn.linear_model.SGDClassifier`.
This is not easily possible for the case of the kernelized SVM.

The second plot visualized the decision surfaces of the RBF kernel SVM and
the linear SVM with approximate kernel maps.
The plot shows decision surfaces of the classifiers projected onto
the first two principal components of the data. This visualization should
be taken with a grain of salt since it is just an interesting slice through
the decision surface in 64 dimensions. In particular note that
a datapoint (represented as a dot) does not necessarily be classified
into the region it is lying in, since it will not lie on the plane
that the first two principal components span.

The usage of :class:`RBFSampler` and :class:`Nystroem` is described in detail
in :ref:`kernel_approximation`.

"""
print(__doc__)

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
#         Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3 clause

# Standard scientific Python imports
import matplotlib.pyplot as plt
import numpy as np
from time import time

# Import datasets, classifiers and performance metrics
from sklearn import datasets, svm, pipeline
from sklearn.kernel_approximation import (RBFSampler,
                                          Nystroem)
from sklearn.decomposition import PCA

# The digits dataset
digits = datasets.load_digits(n_class=9)

# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.data)
data = digits.data / 16.
data -= data.mean(axis=0)

# We learn the digits on the first half of the digits
data_train, targets_train = data[:n_samples / 2], digits.target[:n_samples / 2]

# Now predict the value of the digit on the second half:
data_test, targets_test = data[n_samples / 2:], digits.target[n_samples / 2:]
#data_test = scaler.transform(data_test)

# Create a classifier: a support vector classifier
kernel_svm = svm.SVC(gamma=.2)
linear_svm = svm.LinearSVC()

# create pipeline from kernel approximation
# and linear svm
feature_map_fourier = RBFSampler(gamma=.2, random_state=1)
feature_map_nystroem = Nystroem(gamma=.2, random_state=1)
fourier_approx_svm = pipeline.Pipeline([("feature_map", feature_map_fourier),
                                        ("svm", svm.LinearSVC())])

nystroem_approx_svm = pipeline.Pipeline([("feature_map", feature_map_nystroem),
                                        ("svm", svm.LinearSVC())])

# fit and predict using linear and kernel svm:

kernel_svm_time = time()
kernel_svm.fit(data_train, targets_train)
kernel_svm_score = kernel_svm.score(data_test, targets_test)
kernel_svm_time = time() - kernel_svm_time

linear_svm_time = time()
linear_svm.fit(data_train, targets_train)
linear_svm_score = linear_svm.score(data_test, targets_test)
linear_svm_time = time() - linear_svm_time

sample_sizes = 30 * np.arange(1, 10)
fourier_scores = []
nystroem_scores = []
fourier_times = []
nystroem_times = []

for D in sample_sizes:
    fourier_approx_svm.set_params(feature_map__n_components=D)
    nystroem_approx_svm.set_params(feature_map__n_components=D)
    start = time()
    nystroem_approx_svm.fit(data_train, targets_train)
    nystroem_times.append(time() - start)

    start = time()
    fourier_approx_svm.fit(data_train, targets_train)
    fourier_times.append(time() - start)

    fourier_score = fourier_approx_svm.score(data_test, targets_test)
    nystroem_score = nystroem_approx_svm.score(data_test, targets_test)
    nystroem_scores.append(nystroem_score)
    fourier_scores.append(fourier_score)

# plot the results:
plt.figure(figsize=(8, 8))
accuracy = plt.subplot(211)
# second y axis for timeings
timescale = plt.subplot(212)

accuracy.plot(sample_sizes, nystroem_scores, label="Nystroem approx. kernel")
timescale.plot(sample_sizes, nystroem_times, '--',
               label='Nystroem approx. kernel')

accuracy.plot(sample_sizes, fourier_scores, label="Fourier approx. kernel")
timescale.plot(sample_sizes, fourier_times, '--',
               label='Fourier approx. kernel')

# horizontal lines for exact rbf and linear kernels:
accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [linear_svm_score, linear_svm_score], label="linear svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [linear_svm_time, linear_svm_time], '--', label='linear svm')

accuracy.plot([sample_sizes[0], sample_sizes[-1]],
              [kernel_svm_score, kernel_svm_score], label="rbf svm")
timescale.plot([sample_sizes[0], sample_sizes[-1]],
               [kernel_svm_time, kernel_svm_time], '--', label='rbf svm')

# vertical line for dataset dimensionality = 64
accuracy.plot([64, 64], [0.7, 1], label="n_features")

# legends and labels
accuracy.set_title("Classification accuracy")
timescale.set_title("Training times")
accuracy.set_xlim(sample_sizes[0], sample_sizes[-1])
accuracy.set_xticks(())
accuracy.set_ylim(np.min(fourier_scores), 1)
timescale.set_xlabel("Sampling steps = transformed feature dimension")
accuracy.set_ylabel("Classification accuracy")
timescale.set_ylabel("Training time in seconds")
accuracy.legend(loc='best')
timescale.legend(loc='best')

# visualize the decision surface, projected down to the first
# two principal components of the dataset
pca = PCA(n_components=8).fit(data_train)

X = pca.transform(data_train)

# Gemerate grid along first two principal components
multiples = np.arange(-2, 2, 0.1)
# steps along first component
first = multiples[:, np.newaxis] * pca.components_[0, :]
# steps along second component
second = multiples[:, np.newaxis] * pca.components_[1, :]
# combine
grid = first[np.newaxis, :, :] + second[:, np.newaxis, :]
flat_grid = grid.reshape(-1, data.shape[1])

# title for the plots
titles = ['SVC with rbf kernel',
          'SVC (linear kernel)\n with Fourier rbf feature map\n'
          'n_components=100',
          'SVC (linear kernel)\n with Nystroem rbf feature map\n'
          'n_components=100']

plt.tight_layout()
plt.figure(figsize=(12, 5))

# predict and plot
for i, clf in enumerate((kernel_svm, nystroem_approx_svm,
                         fourier_approx_svm)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, m_max]x[y_min, y_max].
    plt.subplot(1, 3, i + 1)
    Z = clf.predict(flat_grid)

    # Put the result into a color plot
    Z = Z.reshape(grid.shape[:-1])
    plt.contourf(multiples, multiples, Z, cmap=plt.cm.Paired)
    plt.axis('off')

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=targets_train, cmap=plt.cm.Paired)

    plt.title(titles[i])
plt.tight_layout()
plt.show()


import numpy as np

import sklearn.preprocessing as prep
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

from autoencoder.autoencoder_models.DenoisingAutoencoder import MaskingNoiseAutoencoder

mnist = input_data.read_data_sets('MNIST_data', one_hot = True)

def standard_scale(X_train, X_test):
    preprocessor = prep.StandardScaler().fit(X_train)
    X_train = preprocessor.transform(X_train)
    X_test = preprocessor.transform(X_test)
    return X_train, X_test

def get_random_block_from_data(data, batch_size):
    start_index = np.random.randint(0, len(data) - batch_size)
    return data[start_index:(start_index + batch_size)]

X_train, X_test = standard_scale(mnist.train.images, mnist.test.images)

n_samples = int(mnist.train.num_examples)
training_epochs = 100
batch_size = 128
display_step = 1

autoencoder = MaskingNoiseAutoencoder(n_input = 784,
                                      n_hidden = 200,
                                      transfer_function = tf.nn.softplus,
                                      optimizer = tf.train.AdamOptimizer(learning_rate = 0.001),
                                      dropout_probability = 0.95)

for epoch in range(training_epochs):
    avg_cost = 0.
    total_batch = int(n_samples / batch_size)
    for i in range(total_batch):
        batch_xs = get_random_block_from_data(X_train, batch_size)

        cost = autoencoder.partial_fit(batch_xs)

        avg_cost += cost / n_samples * batch_size

    if epoch % display_step == 0:
        print "Epoch:", '%04d' % (epoch + 1), \
            "cost=", "{:.9f}".format(avg_cost)

print "Total cost: " + str(autoencoder.calc_total_cost(X_test))


from epidemic_emulator import node
from datetime import datetime
import platform
import argparse
import time
import os
import matplotlib.pyplot as plt
import random

def parse_network(f, node_id, topology = "clique"):
    neighbors = []
    nd = None
    t = datetime.now()
    t = t-t

    net = []
    index = -1
    cnt = 0

    for i in f:
        i = i.rstrip("\n").split("|")
        if len(i)<4:
            continue
        u = (i[0],(i[1],int(i[2])),[(i[3],t)])
        if i[0]==node_id:
            nd = u
            index = cnt
        net.append(u)
        cnt+=1
    f.close()

    # clique
    if topology == "clique":
        neighbors = [i for i in net if i[0] != node_id]

    # star
    elif topology == "star":
        if index > 0:
            neighbors = [net[0]]
        else:
            neighbors = net[1:]

    return neighbors,nd

def simulation_controller(args,nd,network):

    # Example nd value:
    #('9', ('127.0.0.1', 9179), [('S', datetime.timedelta(0))])
    #
    # network is a tuple containing every node identifier constructed from
    # args.network (default=network.txt) file

    r = args.recovery_rate
    e = args.endogenous_rate
    x = args.exogenous_rate

    if nd is not None:
        with node.Node(r,e,x) as a:

            a.start(nd, network)

            if args.interaction == 1:
                try:
                    help_text = """>> Commands:
        0 (help) -> print this
        1 (print current) -> print current network state
        2 (print history) -> print network history
        3 (end) -> send shutdown message to all nodes
        4 (display state) -> display current network state
        5 (display history) -> display network history
        """
                    print help_text
                    while True:
                        opt = raw_input(">> Insert command: ")
                        if opt == "0":
                            print help_text
                        elif opt == "1":
                            #print a.network_state(),"\n"
                            a.print_state()
                        elif opt == "2":
                            #print a.network_history(),"\n"
                            a.print_history()
                        elif opt == "3":
                            a.display_history()
                            a.network_shutdown()
                            a.stop()
                            break
                        elif opt == "4":
                            a.display_state()
                        elif opt == "5":
                            a.display_history()
                        else:
                            print "Invalid input\n"
                except:
                    a.network_shutdown()
                    a.stop()
                finally:
                    a.network_shutdown()
                    a.stop()

            elif args.interaction > 1:
                print("Running simulation for %d seconds." % args.interaction)
                time.sleep(args.interaction)
                #a.display_history()
                simdata = a.save_simulation_data()
                a.network_shutdown()
                a.stop()
                return simdata

            else:
                try:
                    while not a.stopped():
                        time.sleep(2)
                except:
                    a.stop()
                finally:
                    a.stop()

def process_data(simdata,repetitions,simulation_time):

    simresults = [[-1 for t in range(simulation_time+1)] for x in range(repetitions)]
    print_stuff = 1

    for k in range(repetitions):
        if print_stuff:
            print("")
            print("Run #%d" % (k+1))
            print("time\tinfected count")

        t = 0

        for event in simdata[k]:
            if print_stuff: print("%.2f\t%d" % (event[0],event[1]))
            time = int(event[0])
            infected_count = event[1]

            if time < t:
                continue
            elif t < simulation_time+1:
                if print_stuff: print("* %.2f" % event[0])
                while t <= time:
                    simresults[k][t] = infected_count
                    t = t+1
        while t < simulation_time+1:
            simresults[k][t] = infected_count
            t = t+1

        if print_stuff:
            print("")
            print("Processed output:")
            print("time\tinfected count")
            for t in range(simulation_time+1):
                print("%d\t%d" % (t,simresults[k][t]))

    average_results = [0.0 for t in range(simulation_time+1)]
    for t in range(simulation_time+1):
        for k in range(repetitions):
            average_results[t] = average_results[t] + simresults[k][t]
        average_results[t] = float(average_results[t]) / repetitions

    print(average_results)

    plt.plot(list(range(0,simulation_time+1)),average_results,'-o')
    axes = plt.gca()
    axes.set_xlim([0,simulation_time])
    #axes.set_ylim([0,10])
    plt.xlabel("Seconds")
    plt.ylabel("Infected nodes")
    plt.savefig("average_simulation.pdf")

if __name__ == "__main__":
    dir_path = os.path.dirname(os.path.realpath(__file__))
    dir_path_unix = dir_path.replace("\\","/")
    if (platform.system()!="Windows"): dir_path = dir_path_unix

    parser = argparse.ArgumentParser()
    parser.add_argument("-id","--identifier",required=True,
                        help="Node identifier")
    parser.add_argument("-n","--network",type=argparse.FileType('r'), default = dir_path_unix+"/network.txt",
                        help="File that contains the network's description; each line presents node_id|node_ip|port_number|initial_state")
#    parser.add_argument("-i","--interactive",type=int,default=0,
#                        help="Interactive mode")
    parser.add_argument("-i","--interaction",type=int,default=0,
                        help="Interaction mode: default (0), interactive (1), simulation (2)")
    parser.add_argument("-r","--recovery_rate",type=float,#default=1.0,
                        help="Simulation parameter: recovery_rate")
    parser.add_argument("-e","--endogenous_rate",type=float,#default=1.0,
                        help="Simulation parameter: endogenous_infection_rate")
    parser.add_argument("-x","--exogenous_rate",type=float,#default=1e-6,
                        help="Simulation parameter: exogenous_infection_rate")
    parser.add_argument("-t","--topology",choices=["clique","star"],default="clique",
                        help="Network topology: clique or star")
    args = parser.parse_args()

    network = {}
    if args.network is not None:
        network,nd = parse_network(args.network, args.identifier, args.topology)

    simulation_time = args.interaction

    repetitions = 1
    simdata = []
    for i in range(repetitions):
        simdata.append(simulation_controller(args,nd,network))

    if args.identifier == '0':
        process_data(simdata,repetitions,simulation_time)


# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause
import os
from os.path import join

import numpy

from sklearn._build_utils import get_blas_info

def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration

    cblas_libs, blas_info = get_blas_info()

    libraries = []
    if os.name == 'posix':
        cblas_libs.append('m')
        libraries.append('m')

    config = Configuration('cluster', parent_package, top_path)
    config.add_extension('_dbscan_inner',
                         sources=['_dbscan_inner.cpp'],
                         include_dirs=[numpy.get_include()],
                         language="c++")

    config.add_extension('_hierarchical',
                         sources=['_hierarchical.cpp'],
                         language="c++",
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension(
        '_k_means',
        libraries=cblas_libs,
        sources=['_k_means.c'],
        include_dirs=[join('..', 'src', 'cblas'),
                      numpy.get_include(),
                      blas_info.pop('include_dirs', [])],
        extra_compile_args=blas_info.pop('extra_compile_args', []),
        **blas_info
    )

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())


import matplotlib
from matplotlib import patches as patches

from ...core.util import match_spec
from ...core.options import abbreviated_exception
from .element import ElementPlot

class AnnotationPlot(ElementPlot):
    """
    AnnotationPlot handles the display of all annotation elements.
    """

    def __init__(self, annotation, **params):
        self._annotation = annotation
        super(AnnotationPlot, self).__init__(annotation, **params)
        self.handles['annotations'] = []

    def initialize_plot(self, ranges=None):
        annotation = self.hmap.last
        key = self.keys[-1]
        ranges = self.compute_ranges(self.hmap, key, ranges)
        ranges = match_spec(annotation, ranges)
        axis = self.handles['axis']
        opts = self.style[self.cyclic_index]
        with abbreviated_exception():
            handles = self.draw_annotation(axis, annotation.data, opts)
        self.handles['annotations'] = handles
        return self._finalize_axis(key, ranges=ranges)

    def update_handles(self, key, axis, annotation, ranges, style):
        # Clear all existing annotations
        for element in self.handles['annotations']:
            element.remove()

        with abbreviated_exception():
            self.handles['annotations'] = self.draw_annotation(axis, annotation.data, style)

class VLinePlot(AnnotationPlot):
    "Draw a vertical line on the axis"

    style_opts = ['alpha', 'color', 'linewidth', 'linestyle', 'visible']

    def draw_annotation(self, axis, position, opts):
        return [axis.axvline(position, **opts)]

class HLinePlot(AnnotationPlot):
    "Draw a horizontal line on the axis"

    style_opts = ['alpha', 'color', 'linewidth', 'linestyle', 'visible']

    def draw_annotation(self, axis, position, opts):
        "Draw a horizontal line on the axis"
        return [axis.axhline(position, **opts)]

class TextPlot(AnnotationPlot):
    "Draw the Text annotation object"

    style_opts = ['alpha', 'color', 'family', 'weight', 'rotation', 'fontsize', 'visible']

    def draw_annotation(self, axis, data, opts):
        (x,y, text, fontsize,
         horizontalalignment, verticalalignment, rotation) = data
        opts['fontsize'] = fontsize
        return [axis.text(x,y, text,
                          horizontalalignment = horizontalalignment,
                          verticalalignment = verticalalignment,
                          rotation=rotation, **opts)]

class ArrowPlot(AnnotationPlot):
    "Draw an arrow using the information supplied to the Arrow annotation"

    _arrow_style_opts = ['alpha', 'color', 'lw', 'linewidth', 'visible']
    _text_style_opts = TextPlot.style_opts

    style_opts = sorted(set(_arrow_style_opts + _text_style_opts))

    def draw_annotation(self, axis, data, opts):
        direction, text, xy, points, arrowstyle = data
        arrowprops = dict({'arrowstyle':arrowstyle},
                          **{k: opts[k] for k in self._arrow_style_opts if k in opts})
        textopts = {k: opts[k] for k in self._text_style_opts if k in opts}
        if direction in ['v', '^']:
            xytext = (0, points if direction=='v' else -points)
        elif direction in ['>', '<']:
            xytext = (points if direction=='<' else -points, 0)
        return [axis.annotate(text, xy=xy, textcoords='offset points',
                              xytext=xytext, ha="center", va="center",
                              arrowprops=arrowprops, **textopts)]

class SplinePlot(AnnotationPlot):
    "Draw the supplied Spline annotation (see Spline docstring)"

    style_opts = ['alpha', 'edgecolor', 'linewidth', 'linestyle', 'visible']

    def draw_annotation(self, axis, data, opts):
        verts, codes = data
        patch = patches.PathPatch(matplotlib.path.Path(verts, codes),
                                  facecolor='none', **opts)
        axis.add_patch(patch)
        return [patch]


"""
Electron-phonon coupling self-energy
====================================

A diagram containing loopy lines.
"""
from feynman import Diagram
import matplotlib.pyplot as plt

fig = plt.figure(figsize=(8,2))
ax = fig.add_axes([0,0,1,1], frameon=False)

ax.set_xlim(0, fig.get_size_inches()[0])
ax.set_ylim(0, fig.get_size_inches()[1])

# Init D and ax
D = Diagram(ax)

D.x0 = 0.2
D.y0 = sum(D.ax.get_ylim()) * .35

# Various size
opwidth = 1.
linlen = 2.
txtpad = .8
wiggle_amplitude=.1

# Line styles
Ph_style = dict(style='elliptic loopy', ellipse_spread=.6, xamp=.10, yamp=-.15, nloops=15)
DW_style = dict(style='circular loopy', circle_radius=.7, xamp=.10, yamp=.15, nloops=18)
G_style = dict(style='simple', arrow=True, arrow_param={'width':0.15, 'length': .3})

# Item 1
v11 = D.vertex([D.x0, D.y0])
v12 = D.vertex(v11.xy, dx=opwidth)
Sigma = D.operator([v11, v12])
Sigma.text("$\Sigma^{ep}$")

# Symbol
D.text(v12.x + txtpad, D.y0, "=")

# Item 3
v21 = D.vertex([v12.x + 2 * txtpad,  D.y0 - 0.3])
v22 = D.vertex(v21.xy, dx=linlen)
G = D.line(v21, v22, **G_style)
Ph = D.line(v21, v22, **Ph_style)

# Symbol
D.text(v22.x + txtpad, D.y0, "+")

# Item 3
v31 = D.vertex([v22.x + 3 * txtpad,  D.y0 - 0.3])
DW = D.line(v31, v31, **DW_style)

D.plot()
plt.show()


#! usr/bin/env python

# This script is for testing without having to host the flask app.

import folium
import pandas as pd
import os
from sqlalchemy import create_engine
import geojson

DATABASE_URL = os.environ["DATABASE_URL"]
STATES_GEOJSON_PATH = "static/us-states.json"

engine = create_engine(DATABASE_URL)
with engine.connect() as db:

    query = "Select state, count(*) From registry Group By state;"
    df = pd.read_sql_query(query, db)

with open(STATES_GEOJSON_PATH, "r") as file:
    gj = geojson.load(file)

# Folium choropleth requires a one-to-one correspondence between GeoJSON
#  features (state definitions) and shade values, so we will make a new
#  GeoJSON object that is a FeatureCollection of only the states that we
#  have data for.
relevant_features = [feature for feature in gj["features"]
                     if ("id" in feature) and
                     (feature["id"] in df["state"].values)]

gj_relevant = geojson.FeatureCollection(relevant_features)

geo_str = geojson.dumps(gj_relevant)

base_map = folium.Map([43, -100], zoom_start=5)

base_map.choropleth(
    geo_str=geo_str,
    data=df,
    columns=['state', 'count'],
    key_on='feature.id',
    fill_color='PuBuGn',
)

base_map.save("map.html")


from fonction_py.tools import *
from fonction_py.preprocess import *
from sklearn import linear_model
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import cross_validation
from sklearn.linear_model import LogisticRegression
from sklearn import tree
from sklearn import svm
from sklearn import decomposition
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.grid_search import GridSearchCV
from sklearn.grid_search import RandomizedSearchCV
from scipy.stats import uniform as sp_randint
from sklearn import datasets
from sklearn.linear_model import Ridge
from fonction_py.tim import *

import time

def faireTout():
    fields = ['DATE', 'DAY_OFF', 'WEEK_END', 'DAY_WE_DS', 'ASS_ASSIGNMENT', 'CSPL_RECEIVED_CALLS' ] # selectionne les colonnes à lire
    c = pd.DataFrame()
<<<<<<< HEAD
    listmodel = faireListModel()#recupere le nom et les modeles de chaque truc

    data=pd.read_csv("data/trainPure.csv", sep=";", usecols=fields) # LECTURE du fichier de train,
    resultat = pd.read_csv("data/submission.txt", sep="\t") # LECTURE dufichier de test
    res=[]
    model = listmodel[0]
    for model in listmodel:
        print(model[0]) #affiche le ass assignment
        (xTest, x, souvenir, y)=preprocessTOTAL(model[0]) # ajuste le nombre et le nom de feature pour que xTest et x aient les memes
        mod= GradientBoostingRegressor(loss='huber', alpha=0.9,n_estimators=100, max_depth=3,learning_rate=.1, min_samples_leaf=9,min_samples_split=9)
        mod.fit(x, y) #s'entraine
        pred = mod.predict(xTest) # predit
        pred[pred>max(y)*1.05]=max(y)*1.05 # pour pas predire trop grand
        pred[pred<0]=0 # pas de negatif
        pred =np.round(pred).astype(int) # to int
        souvenir['prediction']=pred # on l'ajoute a souvenir qui garde le format standard et la date pour qu'on remette tout a la bonne place a la fin
        resultat=pd.merge(resultat, souvenir, how='left',on=['DATE', 'ASS_ASSIGNMENT']) # on remet chaque prediction à la bonne ligne -> il cree prediction_x et prediction_y car l'ancienne prediction et la nouvelle colonne de prediction
        resultat=resultat.fillna(0) # on remplit les endroits ou on a pas predit avec des 0
        resultat['prediction'] = resultat['prediction_x']+resultat['prediction_y'] # merge les deux colonnes
        del resultat['prediction_x']
        del resultat['prediction_y']

=======
    listmodel = faireListModel()
    #'Evenements',  'Gestion Amex'
    #setFields = set(pd.read_csv("data/fields.txt", sep=";")['0'].values)
#    resultat = pd.read_csv("data/submission.txt", sep="\t")

    i=0
#    res = []
    start_time = time.time()
    model = listmodel[24]
    data=pd.read_csv("data/trainPure.csv", sep=";", usecols=fields) # LECTURE

    resultat = pd.read_csv("data/submission.txt", sep="\t") # LECTURE
    res=[]
    for model in listmodel:
        i = i+1
        print(model[0])
        x,y = preprocess(data.copy(), model[0]) # rajoute les features
        model[1].fit(x, y)
        #model.score(xTrain, yTrain)
        (xTest, souvenir)=preprocessFINAL(x,model[0])
        pred = model[1].predict(xTest)
        pred[pred>max(y)*1.05]=max(y)*1.05
        pred[pred<0]=0
        pred =np.round(pred)
        souvenir['prediction']=int(pred)
        resultat=pd.merge(resultat, souvenir, how='left',on=['DATE', 'ASS_ASSIGNMENT'])
        resultat=resultat.fillna(0)
        resultat['prediction'] = resultat['prediction_x']+resultat['prediction_y']
        del resultat['prediction_x']
        del resultat['prediction_y']
    x,y = preprocess(data.copy(), 'Téléphonie') # rajoute les features
    #model.score(xTrain, yTrain)
    (xTest, souvenir)=preprocessFINAL(x,'Téléphonie')
    pred=telephoniePred(x,y,xTest)
    pred[pred>max(y)*1.05]=max(y)*1.05
    pred[pred<0]=0
    pred =np.round(pred)
    souvenir['prediction']=int(pred)
    resultat=pd.merge(resultat, souvenir, how='left',on=['DATE', 'ASS_ASSIGNMENT'])
    resultat=resultat.fillna(0)
    resultat['prediction'] = resultat['prediction_x']+resultat['prediction_y']
    del resultat['prediction_x']
    del resultat['prediction_y']
<<<<<<< HEAD
    pd.DataFrame(res).to_csv("reslist.csv", sep=";", decimal=",")
    resultat.to_csv("vraipred.txt", sep="\t", index =False)
=======
>>>>>>> origin/master
    resultat['prediction']=resultat['prediction'].astype(int)
    resultat.to_csv("pouranalyse.txt", sep="\t", index =False, encoding='utf-8')

>>>>>>> origin/master
    return resultat

def faireListModel():
    return [('CAT',  linear_model.LinearRegression()),
    ('CMS', RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=5,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=10, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Crises',linear_model.LinearRegression()),
    ('Domicile', RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=30,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=90, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Gestion',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=30,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Gestion - Accueil Telephonique',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=70, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Gestion Assurances',RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=20,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=20, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Gestion Clients', RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=10,
           max_features=90, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=50, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Gestion DZ', RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=5,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=30, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Gestion Relation Clienteles',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=10,
           max_features=90, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=110, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Gestion Renault', RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=30,
           max_features=50, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=30, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Japon',RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=10,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=30, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Manager',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=10,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=30, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Mécanicien',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Médical',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=30,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Nuit', RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Prestataires',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('RENAULT',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=80,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('RTC',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Regulation Medicale',linear_model.LinearRegression()),
    ('SAP',RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=20,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=30, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Services',RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=30,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=30, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Tech. Axa',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Tech. Inter',RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=30,
           max_features=30, max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=30, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Tech. Total',RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=70,
           max_features='auto', max_leaf_nodes=None, min_samples_leaf=1,
           min_samples_split=2, min_weight_fraction_leaf=0.0,
           n_estimators=100, n_jobs=1, oob_score=False, random_state=None,
           verbose=0, warm_start=False)),
    ('Téléphonie',GradientBoostingRegressor(loss='huber', alpha=0.9,n_estimators=100, max_depth=3,learning_rate=.1, min_samples_leaf=9,min_samples_split=9) )]