import numpy as np
import pandas as pd
# from matplotlib.pyplot import plot,show,draw
import scipy.io
from functions import *
import _pickle as cPickle
import time
import os, sys
import ipyparallel
import neuroseries as nts

data_directory = '/mnt/DataGuillaume/MergedData/'
datasets = np.loadtxt(data_directory+'datasets_ThalHpc.list', delimiter = '\n', dtype = str, comments = '#')

# to know which neurons to keep
theta_mod, theta_ses    = loadThetaMod('/mnt/DataGuillaume/MergedData/THETA_THAL_mod.pickle', datasets, return_index=True)
theta               = pd.DataFrame( index = theta_ses['rem'],
                                    columns = ['phase', 'pvalue', 'kappa'],
                                    data = theta_mod['rem'])
tmp2            = theta.index[theta.isnull().any(1)].values
tmp3            = theta.index[(theta['pvalue'] > 0.01).values].values
tmp             = np.unique(np.concatenate([tmp2,tmp3]))
theta_modth = theta.drop(tmp, axis = 0)
neurons_index = theta_modth.index.values

bins1 = np.arange(-1005, 1010, 25)*1000
times = np.floor(((bins1[0:-1] + (bins1[1] - bins1[0])/2)/1000)).astype('int')
premeanscore = {i:{'rem':pd.DataFrame(index = [], columns = ['mean', 'std']),'rip':pd.DataFrame(index = times, columns = [])} for i in range(3)}# BAD
posmeanscore = {i:{'rem':pd.DataFrame(index = [], columns = ['mean', 'std']),'rip':pd.DataFrame(index = times, columns = [])} for i in range(3)}# BAD
bins2 = np.arange(-1012.5,1025,25)*1000
tsmax = {i:pd.DataFrame(columns = ['pre', 'pos']) for i in range(3)}

clients = ipyparallel.Client()
print(clients.ids)
dview = clients.direct_view()
def compute_pop_pca(session):
    data_directory = '/mnt/DataGuillaume/MergedData/'
    import numpy as np
    import scipy.io
    import scipy.stats
    import _pickle as cPickle
    import time
    import os, sys
    import neuroseries as nts
    from functions import loadShankStructure, loadSpikeData, loadEpoch, loadThetaMod, loadSpeed, loadXML, loadRipples, loadLFP, downsample, getPeaksandTroughs, butter_bandpass_filter
    import pandas as pd

    # to know which neurons to keep
    data_directory = '/mnt/DataGuillaume/MergedData/'
    datasets = np.loadtxt(data_directory+'datasets_ThalHpc.list', delimiter = '\n', dtype = str, comments = '#')
    theta_mod, theta_ses    = loadThetaMod('/mnt/DataGuillaume/MergedData/THETA_THAL_mod.pickle', datasets, return_index=True)
    theta               = pd.DataFrame( index = theta_ses['rem'],
                                        columns = ['phase', 'pvalue', 'kappa'],
                                        data = theta_mod['rem'])
    tmp2            = theta.index[theta.isnull().any(1)].values
    tmp3            = theta.index[(theta['pvalue'] > 0.01).values].values
    tmp             = np.unique(np.concatenate([tmp2,tmp3]))
    theta_modth = theta.drop(tmp, axis = 0)
    neurons_index = theta_modth.index.values

    bins1 = np.arange(-1005, 1010, 25)*1000
    times = np.floor(((bins1[0:-1] + (bins1[1] - bins1[0])/2)/1000)).astype('int')
    premeanscore = {i:{'rem':pd.DataFrame(index = [], columns = ['mean', 'std']),'rip':pd.DataFrame(index = times, columns = [])} for i in range(3)}
    posmeanscore = {i:{'rem':pd.DataFrame(index = [], columns = ['mean', 'std']),'rip':pd.DataFrame(index = times, columns = [])} for i in range(3)}
    bins2 = np.arange(-1012.5,1025,25)*1000
    tsmax = {i:pd.DataFrame(columns = ['pre', 'pos']) for i in range(3)}

# for session in datasets:
# for session in datasets[0:15]:
# for session in ['Mouse12/Mouse12-120815']:
    start_time = time.clock()
    print(session)
    generalinfo     = scipy.io.loadmat(data_directory+session+'/Analysis/GeneralInfo.mat')
    shankStructure  = loadShankStructure(generalinfo)
    if len(generalinfo['channelStructure'][0][0][1][0]) == 2:
        hpc_channel     = generalinfo['channelStructure'][0][0][1][0][1][0][0] - 1
    else:
        hpc_channel     = generalinfo['channelStructure'][0][0][1][0][0][0][0] - 1
    spikes,shank    = loadSpikeData(data_directory+session+'/Analysis/SpikeData.mat', shankStructure['thalamus'])
    wake_ep         = loadEpoch(data_directory+session, 'wake')
    sleep_ep        = loadEpoch(data_directory+session, 'sleep')
    sws_ep          = loadEpoch(data_directory+session, 'sws')
    rem_ep          = loadEpoch(data_directory+session, 'rem')
    sleep_ep        = sleep_ep.merge_close_intervals(threshold=1.e3)
    sws_ep          = sleep_ep.intersect(sws_ep)
    rem_ep          = sleep_ep.intersect(rem_ep)
    speed           = loadSpeed(data_directory+session+'/Analysis/linspeed.mat').restrict(wake_ep)
    speed_ep        = nts.IntervalSet(speed[speed>2.5].index.values[0:-1], speed[speed>2.5].index.values[1:]).drop_long_intervals(26000).merge_close_intervals(50000)
    wake_ep         = wake_ep.intersect(speed_ep).drop_short_intervals(3000000)
    n_channel,fs, shank_to_channel = loadXML(data_directory+session+"/"+session.split("/")[1]+'.xml')
    rip_ep,rip_tsd  = loadRipples(data_directory+session)
    hd_info         = scipy.io.loadmat(data_directory+session+'/Analysis/HDCells.mat')['hdCellStats'][:,-1]
    hd_info_neuron  = np.array([hd_info[n] for n in spikes.keys()])
    all_neurons     = np.array(list(spikes.keys()))
    mod_neurons     = np.array([int(n.split("_")[1]) for n in neurons_index if session.split("/")[1] in n])
    if len(sleep_ep) > 1:
        store           = pd.HDFStore("/mnt/DataGuillaume/population_activity_25ms/"+session.split("/")[1]+".h5")
        # all_pop       = store['allwake']
        pre_pop         = store['presleep']
        pos_pop         = store['postsleep']
        store.close()

        store           = pd.HDFStore("/mnt/DataGuillaume/population_activity_100ms/"+session.split("/")[1]+".h5")
        all_pop         = store['allwake']
        # pre_pop       = store['presleep']
        # pos_pop       = store['postsleep']
        store.close()

        def compute_eigen(popwak):
            popwak = popwak - popwak.mean(0)
            popwak = popwak / (popwak.std(0)+1e-8)
            from sklearn.decomposition import PCA
            pca = PCA(n_components = popwak.shape[1])
            xy = pca.fit_transform(popwak.values)
            pc = pca.explained_variance_ > (1 + np.sqrt(1/(popwak.shape[0]/popwak.shape[1])))**2.0
            eigen = pca.components_[pc]
            lambdaa = pca.explained_variance_[pc]
            return eigen, lambdaa

        def compute_score(ep_pop, eigen, lambdaa, thr):
            ep_pop = ep_pop - ep_pop.mean(0)
            ep_pop = ep_pop / (ep_pop.std(0)+1e-8)
            a = ep_pop.values
            score = np.zeros(len(ep_pop))
            for i in range(len(eigen)):
                 if lambdaa[i] >= thr:
                    score += (np.dot(a, eigen[i])**2.0 - np.dot(a**2.0, eigen[i]**2.0))
            score = nts.Tsd(t = ep_pop.index.values, d = score)
            return score

        def compute_rip_score(tsd, score, bins):
            times = np.floor(((bins[0:-1] + (bins[1] - bins[0])/2)/1000)).astype('int')
            rip_score = pd.DataFrame(index = times, columns = [])
            for r,i in zip(tsd.index.values,range(len(tsd))):
                xbins = (bins + r).astype('int')
                y = score.groupby(pd.cut(score.index.values, bins=xbins, labels = times)).mean()
                if ~y.isnull().any():
                    rip_score[r] = y

            return rip_score

        def get_xmin(ep, minutes):
            duree = (ep['end'] - ep['start'])/1000/1000/60
            tmp = ep.iloc[np.where(np.ceil(duree.cumsum()) <= minutes + 1)[0]]
            return nts.IntervalSet(tmp['start'], tmp['end'])

        pre_ep          = nts.IntervalSet(sleep_ep['start'][0], sleep_ep['end'][0])
        post_ep         = nts.IntervalSet(sleep_ep['start'][1], sleep_ep['end'][1])

        pre_sws_ep      = sws_ep.intersect(pre_ep)
        pos_sws_ep      = sws_ep.intersect(post_ep)
        pre_sws_ep      = get_xmin(pre_sws_ep.iloc[::-1], 30)
        pos_sws_ep      = get_xmin(pos_sws_ep, 30)

        if pre_sws_ep.tot_length('s')/60 > 5.0 and pos_sws_ep.tot_length('s')/60 > 5.0:
            for hd in range(3):
                if hd == 0 or hd == 2:
                    index = np.where(hd_info_neuron == 0)[0]
                elif hd == 1:
                    index = np.where(hd_info_neuron == 1)[0]
                if hd == 0:
                    index = np.intersect1d(index, mod_neurons)
                elif hd == 2:
                    index = np.intersect1d(index, np.setdiff1d(all_neurons, mod_neurons))

                allpop = all_pop[index].copy()
                prepop = nts.TsdFrame(pre_pop[index].copy())
                pospop = nts.TsdFrame(pos_pop[index].copy())
                # prepop25ms = nts.TsdFrame(pre_pop_25ms[index].copy())
                # pospop25ms = nts.TsdFrame(pos_pop_25ms[index].copy())
                if allpop.shape[1] and allpop.shape[1] > 5:
                    eigen,lambdaa   = compute_eigen(allpop)
                    seuil           = 1.2
                    if np.sum(lambdaa > seuil):
                        pre_score       = compute_score(prepop, eigen, lambdaa, seuil)
                        pos_score       = compute_score(pospop, eigen, lambdaa, seuil)

                        prerip_score    = compute_rip_score(rip_tsd.restrict(pre_sws_ep), pre_score, bins1)
                        posrip_score    = compute_rip_score(rip_tsd.restrict(pos_sws_ep), pos_score, bins1)

                        # pre_score_25ms    = compute_score(prepop25ms, eigen)
                        # pos_score_25ms    = compute_score(pospop25ms, eigen)
                        # prerip25ms_score = compute_rip_score(rip_tsd.restrict(pre_ep),  pre_score_25ms, bins2)
                        # posrip25ms_score = compute_rip_score(rip_tsd.restrict(post_ep), pos_score_25ms,  bins2)
                        # prerip25ms_score = prerip25ms_score - prerip25ms_score.mean(0)
                        # posrip25ms_score = posrip25ms_score - posrip25ms_score.mean(0)
                        # prerip25ms_score = prerip25ms_score / prerip25ms_score.std(0)
                        # posrip25ms_score = posrip25ms_score / posrip25ms_score.std(0)
                        # prerip25ms_score = prerip25ms_score.loc[-500:500]
                        # posrip25ms_score = posrip25ms_score.loc[-500:500]
                        # sys.exit()
                        # tmp = pd.concat([pd.DataFrame(prerip25ms_score.idxmax().values, columns = ['pre']),pd.DataFrame(posrip25ms_score.idxmax().values, columns = ['pos'])],axis = 1)
                        # tmp = pd.DataFrame(data = [[prerip25ms_score.mean(1).idxmax(), posrip25ms_score.mean(1).idxmax()]], columns = ['pre', 'pos'])
                        # tsmax[hd] = tsmax[hd].append(tmp, ignore_index = True)

                        premeanscore[hd]['rip'][session] = prerip_score.mean(1)
                        posmeanscore[hd]['rip'][session] = posrip_score.mean(1)

                        # if len(rem_ep.intersect(pre_ep)) and len(rem_ep.intersect(post_ep)):
                        #   premeanscore[hd]['rem'].loc[session,'mean'] = pre_score.restrict(rem_ep.intersect(pre_ep)).mean()
                        #   posmeanscore[hd]['rem'].loc[session,'mean'] = pos_score.restrict(rem_ep.intersect(post_ep)).mean()
                        #   premeanscore[hd]['rem'].loc[session,'std'] =  pre_score.restrict(rem_ep.intersect(pre_ep)).std()
                        #   posmeanscore[hd]['rem'].loc[session,'std'] =  pos_score.restrict(rem_ep.intersect(post_ep)).std()

    return [premeanscore, posmeanscore, tsmax]

# sys.exit()

a = dview.map_sync(compute_pop_pca, datasets)

prescore = {i:pd.DataFrame(index = times) for i in range(3)}
posscore = {i:pd.DataFrame(index = times) for i in range(3)}
for i in range(len(a)):
    for j in range(3):
        if len(a[i][0][j]['rip'].columns):
            s = a[i][0][j]['rip'].columns[0]
            prescore[j][s] = a[i][0][j]['rip']
            posscore[j][s] = a[i][1][j]['rip']

# prescore = premeanscore
# posscore = posmeanscore

from pylab import *
titles = ['non hd mod', 'hd', 'non hd non mod']
figure()
for i in range(3):
    subplot(1,3,i+1)

    times = prescore[i].index.values
    # for s in premeanscore[i]['rip'].index.values:
    #   plot(times, premeanscore[i]['rip'].loc[s].values, linewidth = 0.3, color = 'blue')
    #   plot(times, posmeanscore[i]['rip'].loc[s].values, linewidth = 0.3, color = 'red')
    plot(times, gaussFilt(prescore[i].mean(1).values, (1,)), label = 'pre', color = 'blue', linewidth = 2)
    plot(times, gaussFilt(posscore[i].mean(1).values, (1,)), label = 'post', color = 'red', linewidth = 2)
    legend()
    title(titles[i])

show()

sys.exit()

#########################################
# search for peak in 25 ms array
########################################
tsmax = {i:pd.DataFrame(columns = ['pre', 'pos']) for i in range(2)}
for i in range(len(a)):
    for hd in range(2):
        tsmax[hd] = tsmax[hd].append(a[i][2][hd], ignore_index = True)

from pylab import *
plot(tsmax[0]['pos'], np.ones(len(tsmax[0]['pos'])), 'o')
plot(tsmax[0]['pos'].mean(), [1], '|', markersize = 10)
plot(tsmax[1]['pos'], np.zeros(len(tsmax[1]['pos'])), 'o')
plot(tsmax[1]['pos'].mean(), [0], '|', markersize = 10)

sys.exit()

#########################################
# SAVING
########################################

store = pd.HDFStore("../figures/figures_articles/figure3/pca_analysis_3.h5")
for i,j in zip(range(3),('nohd_mod', 'hd', 'nohd_nomod')):
    store.put(j+'pre_rip', prescore[i])
    store.put(j+'pos_rip', posscore[i])
store.close()

# a = dview.map_sync(compute_population_correlation, datasets[0:15])
# for i in range(len(a)):
#   if type(a[i]) is dict:
#       s = list(a[i].keys())[0]
#       premeanscore.loc[s] = a[i][s]['pre']
#       posmeanscore.loc[s] = a[i][s]['pos']

from pylab import *
titles = ['non hd', 'hd']
figure()
for i in range(2):
    subplot(1,3,i+1)

    times = premeanscore[i]['rip'].columns.values
    # for s in premeanscore[i]['rip'].index.values:
    #   plot(times, premeanscore[i]['rip'].loc[s].values, linewidth = 0.3, color = 'blue')
    #   plot(times, posmeanscore[i]['rip'].loc[s].values, linewidth = 0.3, color = 'red')
    plot(times, gaussFilt(premeanscore[i]['rip'].mean(0).values, (1,)), label = 'pre', color = 'blue', linewidth = 2)
    plot(times, gaussFilt(posmeanscore[i]['rip'].mean(0).values, (1,)),label = 'post', color = 'red', linewidth = 2)
    legend()
    title(titles[i])
subplot(1,3,3)
bar([1,2], [premeanscore[0]['rem'].mean(0)['mean'], premeanscore[1]['rem'].mean(0)['mean']])
bar([3,4], [posmeanscore[0]['rem'].mean(0)['mean'], posmeanscore[1]['rem'].mean(0)['mean']])
xticks([1,2], ['non hd', 'hd'])
xticks([3,4], ['non hd', 'hd'])

show()

figure()
subplot(121)
times = premeanscore[0]['rip'].columns.values
for s in premeanscore[0]['rip'].index.values:
    print(s)
    plot(times, premeanscore[0]['rip'].loc[s].values, linewidth = 1, color = 'blue')
plot(premeanscore[0]['rip'].mean(0))
subplot(122)
for s in posmeanscore[0]['rip'].index.values:
    plot(times, posmeanscore[0]['rip'].loc[s].values, linewidth = 1, color = 'red')
plot(posmeanscore[0]['rip'].mean(0))
show()


"""
The :mod:`sklearn.utils` module includes various utilities.
"""
from collections import Sequence

import numpy as np
from scipy.sparse import issparse
import warnings

from .murmurhash import murmurhash3_32
from .validation import (as_float_array,
                         assert_all_finite, warn_if_not_float,
                         check_random_state, column_or_1d, check_array,
                         check_consistent_length, check_X_y, indexable)
from .class_weight import compute_class_weight
from sklearn.utils.sparsetools import minimum_spanning_tree

__all__ = ["murmurhash3_32", "as_float_array",
           "assert_all_finite", "check_array",
           "warn_if_not_float",
           "check_random_state",
           "compute_class_weight",
           "minimum_spanning_tree",
           "column_or_1d", "safe_indexing",
           "check_consistent_length", "check_X_y", 'indexable']

class deprecated(object):
    """Decorator to mark a function or class as deprecated.

    Issue a warning when the function is called/the class is instantiated and
    adds a warning to the docstring.

    The optional extra argument will be appended to the deprecation message
    and the docstring. Note: to use this with the default value for extra, put
    in an empty of parentheses:

    >>> from sklearn.utils import deprecated
    >>> deprecated() # doctest: +ELLIPSIS
    <sklearn.utils.deprecated object at ...>

    >>> @deprecated()
    ... def some_function(): pass
    """

    # Adapted from http://wiki.python.org/moin/PythonDecoratorLibrary,
    # but with many changes.

    def __init__(self, extra=''):
        """
        Parameters
        ----------
        extra: string
          to be added to the deprecation messages

        """
        self.extra = extra

    def __call__(self, obj):
        if isinstance(obj, type):
            return self._decorate_class(obj)
        else:
            return self._decorate_fun(obj)

    def _decorate_class(self, cls):
        msg = "Class %s is deprecated" % cls.__name__
        if self.extra:
            msg += "; %s" % self.extra

        # FIXME: we should probably reset __new__ for full generality
        init = cls.__init__

        def wrapped(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning)
            return init(*args, **kwargs)
        cls.__init__ = wrapped

        wrapped.__name__ = '__init__'
        wrapped.__doc__ = self._update_doc(init.__doc__)
        wrapped.deprecated_original = init

        return cls

    def _decorate_fun(self, fun):
        """Decorate function fun"""

        msg = "Function %s is deprecated" % fun.__name__
        if self.extra:
            msg += "; %s" % self.extra

        def wrapped(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning)
            return fun(*args, **kwargs)

        wrapped.__name__ = fun.__name__
        wrapped.__dict__ = fun.__dict__
        wrapped.__doc__ = self._update_doc(fun.__doc__)

        return wrapped

    def _update_doc(self, olddoc):
        newdoc = "DEPRECATED"
        if self.extra:
            newdoc = "%s: %s" % (newdoc, self.extra)
        if olddoc:
            newdoc = "%s\n\n%s" % (newdoc, olddoc)
        return newdoc

def safe_mask(X, mask):
    """Return a mask which is safe to use on X.

    Parameters
    ----------
    X : {array-like, sparse matrix}
        Data on which to apply mask.

    mask: array
        Mask to be used on X.

    Returns
    -------
        mask
    """
    mask = np.asarray(mask)
    if np.issubdtype(mask.dtype, np.int):
        return mask

    if hasattr(X, "toarray"):
        ind = np.arange(mask.shape[0])
        mask = ind[mask]
    return mask

def safe_indexing(X, indices):
    """Return items or rows from X using indices.

    Allows simple indexing of lists or arrays.

    Parameters
    ----------
    X : array-like, sparse-matrix, list.
        Data from which to sample rows or items.

    indices : array-like, list
        Indices according to which X will be subsampled.
    """
    if hasattr(X, "iloc"):
        # Pandas Dataframes and Series
        return X.iloc[indices]
    elif hasattr(X, "shape"):
        if hasattr(X, 'take') and (hasattr(indices, 'dtype') and
                                   indices.dtype.kind == 'i'):
            # This is often substantially faster than X[indices]
            return X.take(indices, axis=0)
        else:
            return X[indices]
    else:
        return [X[idx] for idx in indices]

def resample(*arrays, **options):
    """Resample arrays or sparse matrices in a consistent way

    The default strategy implements one step of the bootstrapping
    procedure.

    Parameters
    ----------
    `*arrays` : sequence of arrays or scipy.sparse matrices with same shape[0]

    replace : boolean, True by default
        Implements resampling with replacement. If False, this will implement
        (sliced) random permutations.

    n_samples : int, None by default
        Number of samples to generate. If left to None this is
        automatically set to the first dimension of the arrays.

    random_state : int or RandomState instance
        Control the shuffling for reproducible behavior.

    Returns
    -------
    Sequence of resampled views of the collections. The original arrays are
    not impacted.

    Examples
    --------
    It is possible to mix sparse and dense arrays in the same run::

      >>> X = [[1., 0.], [2., 1.], [0., 0.]]
      >>> y = np.array([0, 1, 2])

      >>> from scipy.sparse import coo_matrix
      >>> X_sparse = coo_matrix(X)

      >>> from sklearn.utils import resample
      >>> X, X_sparse, y = resample(X, X_sparse, y, random_state=0)
      >>> X
      array([[ 1.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> X_sparse                   # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
      <3x2 sparse matrix of type '<... 'numpy.float64'>'
          with 4 stored elements in Compressed Sparse Row format>

      >>> X_sparse.toarray()
      array([[ 1.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> y
      array([0, 1, 0])

      >>> resample(y, n_samples=2, random_state=0)
      array([0, 1])

    See also
    --------
    :class:`sklearn.cross_validation.Bootstrap`
    :func:`sklearn.utils.shuffle`
    """
    random_state = check_random_state(options.pop('random_state', None))
    replace = options.pop('replace', True)
    max_n_samples = options.pop('n_samples', None)
    if options:
        raise ValueError("Unexpected kw arguments: %r" % options.keys())

    if len(arrays) == 0:
        return None

    first = arrays[0]
    n_samples = first.shape[0] if hasattr(first, 'shape') else len(first)

    if max_n_samples is None:
        max_n_samples = n_samples

    if max_n_samples > n_samples:
        raise ValueError("Cannot sample %d out of arrays with dim %d" % (
            max_n_samples, n_samples))

    check_consistent_length(*arrays)
    arrays = [check_array(x, accept_sparse='csr', ensure_2d=False)
              for x in arrays]

    if replace:
        indices = random_state.randint(0, n_samples, size=(max_n_samples,))
    else:
        indices = np.arange(n_samples)
        random_state.shuffle(indices)
        indices = indices[:max_n_samples]

    resampled_arrays = []

    for array in arrays:
        array = array[indices]
        resampled_arrays.append(array)

    if len(resampled_arrays) == 1:
        # syntactic sugar for the unit argument case
        return resampled_arrays[0]
    else:
        return resampled_arrays

def shuffle(*arrays, **options):
    """Shuffle arrays or sparse matrices in a consistent way

    This is a convenience alias to ``resample(*arrays, replace=False)`` to do
    random permutations of the collections.

    Parameters
    ----------
    `*arrays` : sequence of arrays or scipy.sparse matrices with same shape[0]

    random_state : int or RandomState instance
        Control the shuffling for reproducible behavior.

    n_samples : int, None by default
        Number of samples to generate. If left to None this is
        automatically set to the first dimension of the arrays.

    Returns
    -------
    Sequence of shuffled views of the collections. The original arrays are
    not impacted.

    Examples
    --------
    It is possible to mix sparse and dense arrays in the same run::

      >>> X = [[1., 0.], [2., 1.], [0., 0.]]
      >>> y = np.array([0, 1, 2])

      >>> from scipy.sparse import coo_matrix
      >>> X_sparse = coo_matrix(X)

      >>> from sklearn.utils import shuffle
      >>> X, X_sparse, y = shuffle(X, X_sparse, y, random_state=0)
      >>> X
      array([[ 0.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> X_sparse                   # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
      <3x2 sparse matrix of type '<... 'numpy.float64'>'
          with 3 stored elements in Compressed Sparse Row format>

      >>> X_sparse.toarray()
      array([[ 0.,  0.],
             [ 2.,  1.],
             [ 1.,  0.]])

      >>> y
      array([2, 1, 0])

      >>> shuffle(y, n_samples=2, random_state=0)
      array([0, 1])

    See also
    --------
    :func:`sklearn.utils.resample`
    """
    options['replace'] = False
    return resample(*arrays, **options)

def safe_sqr(X, copy=True):
    """Element wise squaring of array-likes and sparse matrices.

    Parameters
    ----------
    X : array like, matrix, sparse matrix

    Returns
    -------
    X ** 2 : element wise square
    """
    X = check_array(X, accept_sparse=['csr', 'csc', 'coo'])
    if issparse(X):
        if copy:
            X = X.copy()
        X.data **= 2
    else:
        if copy:
            X = X ** 2
        else:
            X **= 2
    return X

def gen_batches(n, batch_size):
    """Generator to create slices containing batch_size elements, from 0 to n.

    The last slice may contain less than batch_size elements, when batch_size
    does not divide n.

    Examples
    --------
    >>> from sklearn.utils import gen_batches
    >>> list(gen_batches(7, 3))
    [slice(0, 3, None), slice(3, 6, None), slice(6, 7, None)]
    >>> list(gen_batches(6, 3))
    [slice(0, 3, None), slice(3, 6, None)]
    >>> list(gen_batches(2, 3))
    [slice(0, 2, None)]
    """
    start = 0
    for _ in range(int(n // batch_size)):
        end = start + batch_size
        yield slice(start, end)
        start = end
    if start < n:
        yield slice(start, n)

def gen_even_slices(n, n_packs, n_samples=None):
    """Generator to create n_packs slices going up to n.

    Pass n_samples when the slices are to be used for sparse matrix indexing;
    slicing off-the-end raises an exception, while it works for NumPy arrays.

    Examples
    --------
    >>> from sklearn.utils import gen_even_slices
    >>> list(gen_even_slices(10, 1))
    [slice(0, 10, None)]
    >>> list(gen_even_slices(10, 10))                     #doctest: +ELLIPSIS
    [slice(0, 1, None), slice(1, 2, None), ..., slice(9, 10, None)]
    >>> list(gen_even_slices(10, 5))                      #doctest: +ELLIPSIS
    [slice(0, 2, None), slice(2, 4, None), ..., slice(8, 10, None)]
    >>> list(gen_even_slices(10, 3))
    [slice(0, 4, None), slice(4, 7, None), slice(7, 10, None)]
    """
    start = 0
    for pack_num in range(n_packs):
        this_n = n // n_packs
        if pack_num < n % n_packs:
            this_n += 1
        if this_n > 0:
            end = start + this_n
            if n_samples is not None:
                end = min(n_samples, end)
            yield slice(start, end, None)
            start = end

def tosequence(x):
    """Cast iterable x to a Sequence, avoiding a copy if possible."""
    if isinstance(x, np.ndarray):
        return np.asarray(x)
    elif isinstance(x, Sequence):
        return x
    else:
        return list(x)

class ConvergenceWarning(Warning):
    "Custom warning to capture convergence problems"


#!/bin/env python3

# https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.least_squares.html

"""Solve a curve fitting problem using robust loss function to take care of outliers in the data. Define the
model function as y = a + b * exp(c * t), where t is a predictor variable, y is an observation and a, b, c are
parameters to estimate.
"""

import numpy as np
from scipy.optimize import least_squares

# Function which generates the data with noise and outliers:
def gen_data(x, a, b, c, noise=0, n_outliers=0, random_state=0):
    y = a + b*x + c*x**2

    rnd = np.random.RandomState(random_state)
    error = noise * rnd.randn(x.size)
    outliers = rnd.randint(0, x.size, n_outliers)
    error[outliers] *= 10

    return y + error

# Function for computing residuals:
def resFun(c, x, y):
    return c[0] + c[1] * x + c[2] * x**2  -  y

trueCoefs = [-5, 1, 3]
sigma = 1.5
print("True coefficients: ", trueCoefs)
print("Sigma: ", sigma)

f = np.poly1d(trueCoefs)
xDat = np.linspace(0, 2, 20)
errors = sigma*np.random.normal(size=len(xDat))
yDat = f(xDat) + errors

# Initial estimate of parameters:
# x0 = np.array([1.0, 1.0, 0.0])
x0 = np.array([-4.0, 2.0, 5.0])

# Compute a standard least-squares solution:
res = least_squares(resFun, x0, args=(xDat, yDat))
#print('res: ', res)

print('Success:      ', res.success)
print('Cost:         ', res.cost)
print('Optimality:   ', res.optimality)
print('Coefficients: ', res.x)
print('Grad:         ', res.grad)
print('Residuals:    ', res.fun)

Chi2 = sum(res.fun**2)
redChi2 = Chi2/(len(xDat)-len(res.x))           # Reduced Chi^2 = Chi^2 / (n-m)
print("Chi2: ", Chi2, res.cost*2)
print("Red. Chi2: ", redChi2)

# Plot all the curves. We see that by selecting an appropriate loss we can get estimates close to
# optimal even in the presence of strong outliers. But keep in mind that generally it is recommended to try
# 'soft_l1' or 'huber' losses first (if at all necessary) as the other two options may cause difficulties in
# optimization process.

y_true    = gen_data(xDat, trueCoefs[2], trueCoefs[1], trueCoefs[0])
y_lsq     = gen_data(xDat, *res.x)

print()
#exit()

import matplotlib.pyplot as plt
#plt.style.use('dark_background')        # Invert colours
#plt.plot(xDat, yDat, 'o')
plt.errorbar(xDat, yDat, yerr=errors, fmt='ro')  # Plot red circles with actual error bars
plt.plot(xDat, y_true, 'k', linewidth=2, label='true')
plt.plot(xDat, y_lsq, label='linear loss')

plt.xlabel("t")
plt.ylabel("y")
plt.legend()

plt.tight_layout()
# plt.show()
plt.savefig('scipy.optimize.least_squares.png')                 # Save the plot as png
plt.close()                             # Close the plot in order to start a new one later


# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests NumpySource and PandasSource."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.contrib.learn.python.learn.dataframe.transforms import in_memory_source
from tensorflow.python.client import session
from tensorflow.python.framework import ops
from tensorflow.python.platform import test
from tensorflow.python.training import coordinator
from tensorflow.python.training import queue_runner_impl

# pylint: disable=g-import-not-at-top
try:
  import pandas as pd
  HAS_PANDAS = True
except ImportError:
  HAS_PANDAS = False

def get_rows(array, row_indices):
  rows = [array[i] for i in row_indices]
  return np.vstack(rows)

class NumpySourceTestCase(test.TestCase):

  def testNumpySource(self):
    batch_size = 3
    iterations = 1000
    array = np.arange(32).reshape([16, 2])
    numpy_source = in_memory_source.NumpySource(array, batch_size=batch_size)
    index_column = numpy_source().index
    value_column = numpy_source().value
    cache = {}
    with ops.Graph().as_default():
      value_tensor = value_column.build(cache)
      index_tensor = index_column.build(cache)
      with session.Session() as sess:
        coord = coordinator.Coordinator()
        threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord)
        for i in range(iterations):
          expected_index = [
              j % array.shape[0]
              for j in range(batch_size * i, batch_size * (i + 1))
          ]
          expected_value = get_rows(array, expected_index)
          actual_index, actual_value = sess.run([index_tensor, value_tensor])
          np.testing.assert_array_equal(expected_index, actual_index)
          np.testing.assert_array_equal(expected_value, actual_value)
        coord.request_stop()
        coord.join(threads)

class PandasSourceTestCase(test.TestCase):

  def testPandasFeeding(self):
    if not HAS_PANDAS:
      return
    batch_size = 3
    iterations = 1000
    index = np.arange(100, 132)
    a = np.arange(32)
    b = np.arange(32, 64)
    dataframe = pd.DataFrame({"a": a, "b": b}, index=index)
    pandas_source = in_memory_source.PandasSource(
        dataframe, batch_size=batch_size)
    pandas_columns = pandas_source()
    cache = {}
    with ops.Graph().as_default():
      pandas_tensors = [col.build(cache) for col in pandas_columns]
      with session.Session() as sess:
        coord = coordinator.Coordinator()
        threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord)
        for i in range(iterations):
          indices = [
              j % dataframe.shape[0]
              for j in range(batch_size * i, batch_size * (i + 1))
          ]
          expected_df_indices = dataframe.index[indices]
          expected_rows = dataframe.iloc[indices]
          actual_value = sess.run(pandas_tensors)
          np.testing.assert_array_equal(expected_df_indices, actual_value[0])
          for col_num, col in enumerate(dataframe.columns):
            np.testing.assert_array_equal(expected_rows[col].values,
                                          actual_value[col_num + 1])
        coord.request_stop()
        coord.join(threads)

if __name__ == "__main__":
  test.main()


#!/usr/bin/env python
"""
Design FIR filter coefficients using Parks-McClellan or windowing algorithm
and plot filter transfer function.
Michael Hirsch, Ph.D.

example for PiRadar CW prototype,
writing filter coefficients for use by filters.f90:
./FilterDesign.py 9950 10050 100e3 -L 4096 -m firwin -o cwfir.asc

Refs:
http://www.iowahills.com/5FIRFiltersPage.html
"""
import numpy as np
from pathlib import Path
import scipy.signal as signal
from matplotlib.pyplot import show, figure
from argparse import ArgumentParser
from signal_subspace.plots import plotfilt

try:
    import seaborn as sns

    sns.set_context("talk")
except ImportError:
    pass

def computefir(fc, L: int, ofn, fs: int, method: str):
    """
    bandpass FIR design

    https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.firwin.html
    http://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.remez.html

    L: number of taps

    output:
    b: FIR filter coefficients
    """

    assert len(fc) == 2, "specify lower and upper bandpass filter corner frequencies in Hz."

    if method == "remez":
        b = signal.remez(numtaps=L, bands=[0, 0.9 * fc[0], fc[0], fc[1], 1.1 * fc[1], 0.5 * fs], desired=[0, 1, 0], Hz=fs)
    elif method == "firwin":
        b = signal.firwin(L, [fc[0], fc[1]], window="blackman", pass_zero=False, nyq=fs // 2)
    elif method == "firwin2":
        b = signal.firwin2(
            L,
            [0, fc[0], fc[1], fs // 2],
            [0, 1, 1, 0],
            window="blackman",
            nyq=fs // 2,
            # antisymmetric=True,
        )
    else:
        raise ValueError(f"unknown filter design method {method}")

    if ofn:
        ofn = Path(ofn).expanduser()
        print(f"writing {ofn}")
        # FIXME make binary
        with ofn.open("w") as h:
            h.write(f"{b.size}\n")  # first line is number of coefficients
            b.tofile(h, sep=" ")  # second line is space-delimited coefficents

    return b

def butterplot(fs, fc):
    """
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.butter.html
    """
    b, a = signal.butter(4, 100, "low", analog=True)
    w, h = signal.freqs(b, a)
    ax = figure().gca()
    ax.semilogx(fs * 0.5 / np.pi * w, 20 * np.log10(abs(h)))
    ax.set_title("Butterworth filter frequency response")
    ax.set_xlabel("Frequency [Hz]")
    ax.set_ylabel("Amplitude [dB]")
    ax.grid(which="both", axis="both")
    ax.axvline(fc, color="green")  # cutoff frequency
    ax.set_ylim(-50, 0)

def chebyshevplot(fs):
    """
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.cheby1.html#scipy.signal.cheby1
    """
    b, a = signal.cheby1(4, 5, 100, "high", analog=True)
    w, h = signal.freqs(b, a)

    ax = figure().gca()
    ax.semilogx(w, 20 * np.log10(abs(h)))
    ax.set_title("Chebyshev Type I frequency response (rp=5)")
    ax.set_xlabel("Frequency [radians / second]")
    ax.set_ylabel("Amplitude [dB]")
    ax.grid(which="both", axis="both")
    ax.axvline(100, color="green")  # cutoff frequency
    ax.axhline(-5, color="green")  # rp

def main():
    p = ArgumentParser()
    p.add_argument("fc", help="lower,upper bandpass filter corner frequences [Hz]", nargs=2, type=float)
    p.add_argument("fs", help="optional sampling frequency [Hz]", type=float)
    p.add_argument("-o", "--ofn", help="output coefficient file to write")
    p.add_argument("-L", help="number of coefficients for FIR filter", type=int, default=63)
    p.add_argument("-m", "--method", help="filter design method [remez,firwin,firwin2]", default="firwin")
    p.add_argument("-k", "--filttype", help="filter type: low, high, bandpass", default="low")
    p = p.parse_args()

    b = computefir(p.fc, p.L, p.ofn, p.fs, p.method)

    plotfilt(b, p.fs, p.ofn)

    show()

if __name__ == "__main__":
    main()


r"""
Util functions (:mod: `qiita_db.meta_util`)
===========================================

..currentmodule:: qiita_db.meta_util

This module provides utility functions that use the ORM objects. ORM objects
CANNOT import from this file.

Methods
-------

..autosummary::
    :toctree: generated/

    get_lat_longs
"""
# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from os import stat, rename
from os.path import join, relpath, basename
from time import strftime, localtime
import matplotlib.pyplot as plt
import matplotlib as mpl
from base64 import b64encode
from urllib.parse import quote
from io import BytesIO
from datetime import datetime
from collections import defaultdict, Counter
from tarfile import open as topen, TarInfo
from hashlib import md5
from re import sub
from json import loads, dump, dumps

from qiita_db.util import create_nested_path
from qiita_core.qiita_settings import qiita_config, r_client
from qiita_core.configuration_manager import ConfigurationManager
import qiita_db as qdb

def _get_data_fpids(constructor, object_id):
    """Small function for getting filepath IDS associated with data object

    Parameters
    ----------
    constructor : a subclass of BaseData
        E.g., RawData, PreprocessedData, or ProcessedData
    object_id : int
        The ID of the data object

    Returns
    -------
    set of int
    """
    with qdb.sql_connection.TRN:
        obj = constructor(object_id)
        return {fpid for fpid, _, _ in obj.get_filepaths()}

def validate_filepath_access_by_user(user, filepath_id):
    """Validates if the user has access to the filepath_id

    Parameters
    ----------
    user : User object
        The user we are interested in
    filepath_id : int
        The filepath id

    Returns
    -------
    bool
        If the user has access or not to the filepath_id

    Notes
    -----
    Admins have access to all files so True is always returned
    """
    TRN = qdb.sql_connection.TRN
    with TRN:
        if user.level == "admin":
            # admins have access all files
            return True

        sql = """SELECT
            (SELECT array_agg(artifact_id)
             FROM qiita.artifact_filepath
             WHERE filepath_id = {0}) AS artifact,
            (SELECT array_agg(study_id)
             FROM qiita.sample_template_filepath
             WHERE filepath_id = {0}) AS sample_info,
            (SELECT array_agg(prep_template_id)
             FROM qiita.prep_template_filepath
             WHERE filepath_id = {0}) AS prep_info,
            (SELECT array_agg(analysis_id)
             FROM qiita.analysis_filepath
             WHERE filepath_id = {0}) AS analysis""".format(filepath_id)
        TRN.add(sql)

        arid, sid, pid, anid = TRN.execute_fetchflatten()

        # artifacts
        if arid:
            # [0] cause we should only have 1
            artifact = qdb.artifact.Artifact(arid[0])

            if artifact.visibility == 'public':
                # TODO: https://github.com/biocore/qiita/issues/1724
                if artifact.artifact_type in ['SFF', 'FASTQ', 'FASTA',
                                              'FASTA_Sanger',
                                              'per_sample_FASTQ']:
                    study = artifact.study
                    has_access = study.has_access(user, no_public=True)
                    if (not study.public_raw_download and not has_access):
                        return False
                return True
            else:
                study = artifact.study
                if study:
                    # let's take the visibility via the Study
                    return artifact.study.has_access(user)
                else:
                    analysis = artifact.analysis
                    return analysis in (
                        user.private_analyses | user.shared_analyses)
        # sample info files
        elif sid:
            # the visibility of the sample info file is given by the
            # study visibility
            # [0] cause we should only have 1
            return qdb.study.Study(sid[0]).has_access(user)
        # prep info files
        elif pid:
            # the prep access is given by it's artifacts, if the user has
            # access to any artifact, it should have access to the prep
            # [0] cause we should only have 1
            pt = qdb.metadata_template.prep_template.PrepTemplate(
                pid[0])
            a = pt.artifact
            # however, the prep info file could not have any artifacts attached
            # , in that case we will use the study access level
            if a is None:
                return qdb.study.Study(pt.study_id).has_access(user)
            else:
                if (a.visibility == 'public' or a.study.has_access(user)):
                    return True
                else:
                    for c in a.descendants.nodes():
                        if ((c.visibility == 'public' or
                             c.study.has_access(user))):
                            return True
            return False
        # analyses
        elif anid:
            # [0] cause we should only have 1
            aid = anid[0]
            analysis = qdb.analysis.Analysis(aid)
            return analysis in (
                user.private_analyses | user.shared_analyses)
        return False

def update_redis_stats():
    """Generate the system stats and save them in redis

    Returns
    -------
    list of str
        artifact filepaths that are not present in the file system
    """
    STUDY = qdb.study.Study

    number_studies = {'public': 0, 'private': 0, 'sandbox': 0}
    number_of_samples = {'public': 0, 'private': 0, 'sandbox': 0}
    num_studies_ebi = 0
    num_samples_ebi = 0
    number_samples_ebi_prep = 0
    stats = []
    missing_files = []
    per_data_type_stats = Counter()
    for study in STUDY.iter():
        st = study.sample_template
        if st is None:
            continue

        # counting samples submitted to EBI-ENA
        len_samples_ebi = sum([esa is not None
                               for esa in st.ebi_sample_accessions.values()])
        if len_samples_ebi != 0:
            num_studies_ebi += 1
            num_samples_ebi += len_samples_ebi

        samples_status = defaultdict(set)
        for pt in study.prep_templates():
            pt_samples = list(pt.keys())
            pt_status = pt.status
            if pt_status == 'public':
                per_data_type_stats[pt.data_type()] += len(pt_samples)
            samples_status[pt_status].update(pt_samples)
            # counting experiments (samples in preps) submitted to EBI-ENA
            number_samples_ebi_prep += sum([
                esa is not None
                for esa in pt.ebi_experiment_accessions.values()])

        # counting studies
        if 'public' in samples_status:
            number_studies['public'] += 1
        elif 'private' in samples_status:
            number_studies['private'] += 1
        else:
            # note that this is a catch all for other status; at time of
            # writing there is status: awaiting_approval
            number_studies['sandbox'] += 1

        # counting samples; note that some of these lines could be merged with
        # the block above but I decided to split it in 2 for clarity
        if 'public' in samples_status:
            number_of_samples['public'] += len(samples_status['public'])
        if 'private' in samples_status:
            number_of_samples['private'] += len(samples_status['private'])
        if 'sandbox' in samples_status:
            number_of_samples['sandbox'] += len(samples_status['sandbox'])

        # processing filepaths
        for artifact in study.artifacts():
            for adata in artifact.filepaths:
                try:
                    s = stat(adata['fp'])
                except OSError:
                    missing_files.append(adata['fp'])
                else:
                    stats.append(
                        (adata['fp_type'], s.st_size, strftime('%Y-%m',
                         localtime(s.st_mtime))))

    num_users = qdb.util.get_count('qiita.qiita_user')
    num_processing_jobs = qdb.util.get_count('qiita.processing_job')

    lat_longs = dumps(get_lat_longs())

    summary = {}
    all_dates = []
    # these are some filetypes that are too small to plot alone so we'll merge
    # in other
    group_other = {'html_summary', 'tgz', 'directory', 'raw_fasta', 'log',
                   'biom', 'raw_sff', 'raw_qual', 'qza', 'html_summary_dir',
                   'qza', 'plain_text', 'raw_barcodes'}
    for ft, size, ym in stats:
        if ft in group_other:
            ft = 'other'
        if ft not in summary:
            summary[ft] = {}
        if ym not in summary[ft]:
            summary[ft][ym] = 0
            all_dates.append(ym)
        summary[ft][ym] += size
    all_dates = sorted(set(all_dates))

    # sorting summaries
    ordered_summary = {}
    for dt in summary:
        new_list = []
        current_value = 0
        for ad in all_dates:
            if ad in summary[dt]:
                current_value += summary[dt][ad]
            new_list.append(current_value)
        ordered_summary[dt] = new_list

    plot_order = sorted([(k, ordered_summary[k][-1]) for k in ordered_summary],
                        key=lambda x: x[1])

    # helper function to generate y axis, modified from:
    # http://stackoverflow.com/a/1094933
    def sizeof_fmt(value, position):
        number = None
        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
            if abs(value) < 1024.0:
                number = "%3.1f%s" % (value, unit)
                break
            value /= 1024.0
        if number is None:
            number = "%.1f%s" % (value, 'Yi')
        return number

    all_dates_axis = range(len(all_dates))
    plt.locator_params(axis='y', nbins=10)
    plt.figure(figsize=(20, 10))
    for k, v in plot_order:
        plt.plot(all_dates_axis, ordered_summary[k], linewidth=2, label=k)

    plt.xticks(all_dates_axis, all_dates)
    plt.legend()
    plt.grid()
    ax = plt.gca()
    ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(sizeof_fmt))
    plt.xticks(rotation=90)
    plt.xlabel('Date')
    plt.ylabel('Storage space per data type')

    plot = BytesIO()
    plt.savefig(plot, format='png')
    plot.seek(0)
    img = 'data:image/png;base64,' + quote(b64encode(plot.getbuffer()))

    time = datetime.now().strftime('%m-%d-%y %H:%M:%S')

    portal = qiita_config.portal
    # making sure per_data_type_stats has some data so hmset doesn't fail
    if per_data_type_stats == {}:
        per_data_type_stats['No data'] = 0

    vals = [
        ('number_studies', number_studies, r_client.hmset),
        ('number_of_samples', number_of_samples, r_client.hmset),
        ('per_data_type_stats', dict(per_data_type_stats), r_client.hmset),
        ('num_users', num_users, r_client.set),
        ('lat_longs', (lat_longs), r_client.set),
        ('num_studies_ebi', num_studies_ebi, r_client.set),
        ('num_samples_ebi', num_samples_ebi, r_client.set),
        ('number_samples_ebi_prep', number_samples_ebi_prep, r_client.set),
        ('img', img, r_client.set),
        ('time', time, r_client.set),
        ('num_processing_jobs', num_processing_jobs, r_client.set)]
    for k, v, f in vals:
        redis_key = '%s:stats:%s' % (portal, k)
        # important to "flush" variables to avoid errors
        r_client.delete(redis_key)
        f(redis_key, v)

    # preparing vals to insert into DB
    vals = dumps(dict([x[:-1] for x in vals]))
    sql = """INSERT INTO qiita.stats_daily (stats, stats_timestamp)
             VALUES (%s, NOW())"""
    qdb.sql_connection.perform_as_transaction(sql, [vals])

    return missing_files

def get_lat_longs():
    """Retrieve the latitude and longitude of all the public samples in the DB

    Returns
    -------
    list of [float, float]
        The latitude and longitude for each sample in the database
    """
    with qdb.sql_connection.TRN:
        # getting all the public studies
        studies = qdb.study.Study.get_by_status('public')

        results = []
        if studies:
            # we are going to create multiple union selects to retrieve the
            # latigute and longitude of all available studies. Note that
            # UNION in PostgreSQL automatically removes duplicates
            sql_query = """
                SELECT {0}, CAST(sample_values->>'latitude' AS FLOAT),
                       CAST(sample_values->>'longitude' AS FLOAT)
                FROM qiita.sample_{0}
                WHERE sample_values->>'latitude' != 'NaN' AND
                      sample_values->>'longitude' != 'NaN' AND
                      isnumeric(sample_values->>'latitude') AND
                      isnumeric(sample_values->>'longitude')"""
            sql = [sql_query.format(s.id) for s in studies]
            sql = ' UNION '.join(sql)
            qdb.sql_connection.TRN.add(sql)

            # note that we are returning set to remove duplicates
            results = qdb.sql_connection.TRN.execute_fetchindex()

        return results

def generate_biom_and_metadata_release(study_status='public'):
    """Generate a list of biom/meatadata filepaths and a tgz of those files

    Parameters
    ----------
    study_status : str, optional
        The study status to search for. Note that this should always be set
        to 'public' but having this exposed helps with testing. The other
        options are 'private' and 'sandbox'
    """
    studies = qdb.study.Study.get_by_status(study_status)
    qiita_config = ConfigurationManager()
    working_dir = qiita_config.working_dir
    portal = qiita_config.portal
    bdir = qdb.util.get_db_files_base_dir()
    time = datetime.now().strftime('%m-%d-%y %H:%M:%S')

    data = []
    for s in studies:
        # [0] latest is first, [1] only getting the filepath
        sample_fp = relpath(s.sample_template.get_filepaths()[0][1], bdir)

        for a in s.artifacts(artifact_type='BIOM'):
            if a.processing_parameters is None or a.visibility != study_status:
                continue

            merging_schemes, parent_softwares = a.merging_scheme
            software = a.processing_parameters.command.software
            software = '%s v%s' % (software.name, software.version)

            for x in a.filepaths:
                if x['fp_type'] != 'biom' or 'only-16s' in x['fp']:
                    continue
                fp = relpath(x['fp'], bdir)
                for pt in a.prep_templates:
                    categories = pt.categories()
                    platform = ''
                    target_gene = ''
                    if 'platform' in categories:
                        platform = ', '.join(
                            set(pt.get_category('platform').values()))
                    if 'target_gene' in categories:
                        target_gene = ', '.join(
                            set(pt.get_category('target_gene').values()))
                    for _, prep_fp in pt.get_filepaths():
                        if 'qiime' not in prep_fp:
                            break
                    prep_fp = relpath(prep_fp, bdir)
                    # format: (biom_fp, sample_fp, prep_fp, qiita_artifact_id,
                    #          platform, target gene, merging schemes,
                    #          artifact software/version,
                    #          parent sofware/version)
                    data.append((fp, sample_fp, prep_fp, a.id, platform,
                                 target_gene, merging_schemes, software,
                                 parent_softwares))

    # writing text and tgz file
    ts = datetime.now().strftime('%m%d%y-%H%M%S')
    tgz_dir = join(working_dir, 'releases')
    create_nested_path(tgz_dir)
    tgz_name = join(tgz_dir, '%s-%s-building.tgz' % (portal, study_status))
    tgz_name_final = join(tgz_dir, '%s-%s.tgz' % (portal, study_status))
    txt_lines = [
        "biom fp\tsample fp\tprep fp\tqiita artifact id\tplatform\t"
        "target gene\tmerging scheme\tartifact software\tparent software"]
    with topen(tgz_name, "w|gz") as tgz:
        for biom_fp, sample_fp, prep_fp, aid, pform, tg, ms, asv, psv in data:
            txt_lines.append("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (
                biom_fp, sample_fp, prep_fp, aid, pform, tg, ms, asv, psv))
            tgz.add(join(bdir, biom_fp), arcname=biom_fp, recursive=False)
            tgz.add(join(bdir, sample_fp), arcname=sample_fp, recursive=False)
            tgz.add(join(bdir, prep_fp), arcname=prep_fp, recursive=False)
        info = TarInfo(name='%s-%s-%s.txt' % (portal, study_status, ts))
        txt_hd = BytesIO()
        txt_hd.write(bytes('\n'.join(txt_lines), 'ascii'))
        txt_hd.seek(0)
        info.size = len(txt_hd.read())
        txt_hd.seek(0)
        tgz.addfile(tarinfo=info, fileobj=txt_hd)

    with open(tgz_name, "rb") as f:
        md5sum = md5()
        for c in iter(lambda: f.read(4096), b""):
            md5sum.update(c)

    rename(tgz_name, tgz_name_final)

    vals = [
        ('filepath', tgz_name_final[len(working_dir):], r_client.set),
        ('md5sum', md5sum.hexdigest(), r_client.set),
        ('time', time, r_client.set)]
    for k, v, f in vals:
        redis_key = '%s:release:%s:%s' % (portal, study_status, k)
        # important to "flush" variables to avoid errors
        r_client.delete(redis_key)
        f(redis_key, v)

def generate_plugin_releases():
    """Generate releases for plugins
    """
    ARCHIVE = qdb.archive.Archive
    qiita_config = ConfigurationManager()
    working_dir = qiita_config.working_dir

    commands = [c for s in qdb.software.Software.iter(active=True)
                for c in s.commands if c.post_processing_cmd is not None]

    tnow = datetime.now()
    ts = tnow.strftime('%m%d%y-%H%M%S')
    tgz_dir = join(working_dir, 'releases', 'archive')
    create_nested_path(tgz_dir)
    tgz_dir_release = join(tgz_dir, ts)
    create_nested_path(tgz_dir_release)
    for cmd in commands:
        cmd_name = cmd.name
        mschemes = [v for _, v in ARCHIVE.merging_schemes().items()
                    if cmd_name in v]
        for ms in mschemes:
            ms_name = sub('[^0-9a-zA-Z]+', '', ms)
            ms_fp = join(tgz_dir_release, ms_name)
            create_nested_path(ms_fp)

            pfp = join(ms_fp, 'archive.json')
            archives = {k: loads(v)
                        for k, v in ARCHIVE.retrieve_feature_values(
                              archive_merging_scheme=ms).items()
                        if v != ''}
            with open(pfp, 'w') as f:
                dump(archives, f)

            # now let's run the post_processing_cmd
            ppc = cmd.post_processing_cmd

            # concatenate any other parameters into a string
            params = ' '.join(["%s=%s" % (k, v) for k, v in
                              ppc['script_params'].items()])
            # append archives file and output dir parameters
            params = ("%s --fp_archive=%s --output_dir=%s" % (
                params, pfp, ms_fp))

            ppc_cmd = "%s %s %s" % (
                ppc['script_env'], ppc['script_path'], params)
            p_out, p_err, rv = qdb.processing_job._system_call(ppc_cmd)
            p_out = p_out.rstrip()
            if rv != 0:
                raise ValueError('Error %d: %s' % (rv, p_out))
            p_out = loads(p_out)

    # tgz-ing all files
    tgz_name = join(tgz_dir, 'archive-%s-building.tgz' % ts)
    tgz_name_final = join(tgz_dir, 'archive.tgz')
    with topen(tgz_name, "w|gz") as tgz:
        tgz.add(tgz_dir_release, arcname=basename(tgz_dir_release))
    # getting the release md5
    with open(tgz_name, "rb") as f:
        md5sum = md5()
        for c in iter(lambda: f.read(4096), b""):
            md5sum.update(c)
    rename(tgz_name, tgz_name_final)
    vals = [
        ('filepath', tgz_name_final[len(working_dir):], r_client.set),
        ('md5sum', md5sum.hexdigest(), r_client.set),
        ('time', tnow.strftime('%m-%d-%y %H:%M:%S'), r_client.set)]
    for k, v, f in vals:
        redis_key = 'release-archive:%s' % k
        # important to "flush" variables to avoid errors
        r_client.delete(redis_key)
        f(redis_key, v)


"""
============================
Faces dataset decompositions
============================

This example applies to :ref:`olivetti_faces` different unsupervised
matrix decomposition (dimension reduction) methods from the module
:py:mod:`sklearn.decomposition` (see the documentation chapter
:ref:`decompositions`) .

"""
print(__doc__)

# Authors: Vlad Niculae, Alexandre Gramfort
# License: BSD 3 clause

import logging
from time import time

from numpy.random import RandomState
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_olivetti_faces
from sklearn.cluster import MiniBatchKMeans
from sklearn import decomposition

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')
n_row, n_col = 2, 3
n_components = n_row * n_col
image_shape = (64, 64)
rng = RandomState(0)

###############################################################################
# Load faces data
dataset = fetch_olivetti_faces(shuffle=True, random_state=rng)
faces = dataset.data

n_samples, n_features = faces.shape

# global centering
faces_centered = faces - faces.mean(axis=0)

# local centering
faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)

print("Dataset consists of %d faces" % n_samples)

###############################################################################
def plot_gallery(title, images, n_col=n_col, n_row=n_row):
    plt.figure(figsize=(2. * n_col, 2.26 * n_row))
    plt.suptitle(title, size=16)
    for i, comp in enumerate(images):
        plt.subplot(n_row, n_col, i + 1)
        vmax = max(comp.max(), -comp.min())
        plt.imshow(comp.reshape(image_shape), cmap=plt.cm.gray,
                   interpolation='nearest',
                   vmin=-vmax, vmax=vmax)
        plt.xticks(())
        plt.yticks(())
    plt.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.)

###############################################################################
# List of the different estimators, whether to center and transpose the
# problem, and whether the transformer uses the clustering API.
estimators = [
    ('Eigenfaces - RandomizedPCA',
     decomposition.RandomizedPCA(n_components=n_components, whiten=True),
     True),

    ('Non-negative components - NMF',
     decomposition.NMF(n_components=n_components, init='nndsvda', tol=5e-3),
     False),

    ('Independent components - FastICA',
     decomposition.FastICA(n_components=n_components, whiten=True),
     True),

    ('Sparse comp. - MiniBatchSparsePCA',
     decomposition.MiniBatchSparsePCA(n_components=n_components, alpha=0.8,
                                      n_iter=100, batch_size=3,
                                      random_state=rng),
     True),

    ('MiniBatchDictionaryLearning',
        decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                  n_iter=50, batch_size=3,
                                                  random_state=rng),
     True),

    ('Cluster centers - MiniBatchKMeans',
        MiniBatchKMeans(n_clusters=n_components, tol=1e-3, batch_size=20,
                        max_iter=50, random_state=rng),
     True),

    ('Factor Analysis components - FA',
     decomposition.FactorAnalysis(n_components=n_components, max_iter=2),
     True),
]

###############################################################################
# Plot a sample of the input data

plot_gallery("First centered Olivetti faces", faces_centered[:n_components])

###############################################################################
# Do the estimation and plot it

for name, estimator, center in estimators:
    print("Extracting the top %d %s..." % (n_components, name))
    t0 = time()
    data = faces
    if center:
        data = faces_centered
    estimator.fit(data)
    train_time = (time() - t0)
    print("done in %0.3fs" % train_time)
    if hasattr(estimator, 'cluster_centers_'):
        components_ = estimator.cluster_centers_
    else:
        components_ = estimator.components_
    if hasattr(estimator, 'noise_variance_'):
        plot_gallery("Pixelwise variance",
                     estimator.noise_variance_.reshape(1, -1), n_col=1,
                     n_row=1)
    plot_gallery('%s - Train time %.1fs' % (name, train_time),
                 components_[:n_components])

plt.show()