#### embedding
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE, MDS
from mpl_toolkits.mplot3d import Axes3D

class embeddings(obj):
    def __init__(self, model,data):
        self.modelChoice = model
        self.data = data
        # self.data = FeatureMtx_norm

    def PCA_embedding(self,n_components):
        print 'PCA projecting...'
        self.pca = PCA(n_components= n_components,whiten=False)
        self.embedding_ = self.model.fit(data)

        # self.pca = PCAembedding(self.data,50)
        # FeatureAfterPCA = self.pca.transform(self.data)

    def TSNE_embedding(self,n_components):
        # tsne = TSNE(n_components=2, perplexity=30.0)
        tsne3 = TSNE(n_components=n_components, perplexity=30.0)

        # tsne_data = tsne.fit_transform(FeatureAfterPCA50)
        tsne3_data = tsne3.fit_transform(FeatureAfterPCA50)
        # pickle.dump(tsne_data,open(DataPathobj.DataPath+'/tsne_data.p','wb'))
        # tsne_data = pickle.load(open(DataPathobj.DataPath+'/tsne_data.p','rb'))
        self.embedding_ = tsne3_data

    def MDS_embedding(self,n_components):
        self.mds = MDS(n_components=n_components, max_iter=100, n_init=1)
        MDS_data = self.mds.fit_transform(FeatureAfterPCA50)

    def LLE_embedding(self):
        """locally linear embedding_"""
        # self.lle = sklearn.manifold.LocallyLinearEmbedding(n_neighbors=5, n_components=self.n_dimension, reg=0.001, eigen_solver='auto', tol=1e-06, max_iter=100,
        #     method='standard', hessian_tol=0.0001, modified_tol=1e-12, neighbors_algorithm='auto', random_state=None)
        # self.embedding_ = self.lle.fit_transform(data_sampl_*feature_)

        """use DPGMM or Spectral labels"""
        sscfile = loadmat(DataPathobj.sscpath+'001.mat')
        labels_DPGMM = csr_matrix(sscfile['labels_DPGMM_upup'], shape=sscfile['labels_DPGMM_upup'].shape).toarray()
        labels_spectral = csr_matrix(sscfile['labels_spectral_upup'], shape=sscfile['labels_spectral_upup'].shape).toarray()
        trjID = csr_matrix(sscfile['trjID_upup'], shape=sscfile['trjID_upup'].shape).toarray()

        """use connected_components labels"""
        adjfile = loadmat(DataPathobj.adjpath+'20knn&thresh_Gaussian_diff_dir_001.mat')
        labels_CC = csr_matrix(adjfile['c_upup'], shape=adjfile['c_upup'].shape).toarray()

        """use fake ground truth labels"""
        arrange_index = pickle.load(open(DataPathobj.DataPath+'/arrange_index.p','rb'))

        # labels_fakeGT = labels_CC[arrange_index]
        labels_fakeGT = np.zeros_like(labels_CC)
        for ii in range(0,int(labels_fakeGT.shape[1]/20),1):
            labels_fakeGT[0,arrange_index[20*ii:min(20*(ii+1),labels_fakeGT.shape[1])]] = ii
            # labels_fakeGT[0,5*ii:min(5*(ii+1),labels_fakeGT.shape[1])] = ii

    def visEmbedding(self):
        # fig = plt.figure()
        # ax = fig.add_subplot(111, projection='3d')

        # labels = labels_DPGMM
        # labels = labels_spectral
        # labels = labels_CC
        labels = labels_fakeGT
        # data = MDS_data
        data = tsne_data

        clustered_color = np.array([np.random.randint(0,255) for _ in range(3*int(len(np.unique(labels))))]).reshape(len(np.unique(labels)),3)
        plt.figure()
        for ii in range(labels.shape[1]):
            plt.scatter(data[ii,0],data[ii,1],color=(clustered_color[int(labels[0,ii])].T/255.0))
        plt.draw()


import scipy.sparse as sp
import numpy as np
import sys
from sklearn.externals.six.moves import cStringIO as StringIO

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.utils.testing import assert_raises_regex, assert_true
from sklearn.utils.estimator_checks import check_estimator
from sklearn.utils.estimator_checks import check_estimators_unfitted
from sklearn.linear_model import LogisticRegression
from sklearn.utils.validation import check_X_y, check_array

class CorrectNotFittedError(ValueError):
    """Exception class to raise if estimator is used before fitting.

    Like NotFittedError, it inherits from ValueError, but not from
    AttributeError. Used for testing only.
    """

class BaseBadClassifier(BaseEstimator, ClassifierMixin):
    def fit(self, X, y):
        return self

    def predict(self, X):
        return np.ones(X.shape[0])

class NoCheckinPredict(BaseBadClassifier):
    def fit(self, X, y):
        X, y = check_X_y(X, y)
        return self

class NoSparseClassifier(BaseBadClassifier):
    def fit(self, X, y):
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc'])
        if sp.issparse(X):
            raise ValueError("Nonsensical Error")
        return self

    def predict(self, X):
        X = check_array(X)
        return np.ones(X.shape[0])

class CorrectNotFittedErrorClassifier(BaseBadClassifier):
    def fit(self, X, y):
        X, y = check_X_y(X, y)
        self.coef_ = np.ones(X.shape[1])
        return self

    def predict(self, X):
        if not hasattr(self, 'coef_'):
            raise CorrectNotFittedError("estimator is not fitted yet")
        X = check_array(X)
        return np.ones(X.shape[0])

def test_check_estimator():
    # tests that the estimator actually fails on "bad" estimators.
    # not a complete test of all checks, which are very extensive.

    # check that we have a set_params and can clone
    msg = "it does not implement a 'get_params' methods"
    assert_raises_regex(TypeError, msg, check_estimator, object)
    # check that we have a fit method
    msg = "object has no attribute 'fit'"
    assert_raises_regex(AttributeError, msg, check_estimator, BaseEstimator)
    # check that fit does input validation
    msg = "TypeError not raised by fit"
    assert_raises_regex(AssertionError, msg, check_estimator, BaseBadClassifier)
    # check that predict does input validation (doesn't accept dicts in input)
    msg = "Estimator doesn't check for NaN and inf in predict"
    assert_raises_regex(AssertionError, msg, check_estimator, NoCheckinPredict)
    # check for sparse matrix input handling
    msg = "Estimator type doesn't seem to fail gracefully on sparse data"
    # the check for sparse input handling prints to the stdout,
    # instead of raising an error, so as not to remove the original traceback.
    # that means we need to jump through some hoops to catch it.
    old_stdout = sys.stdout
    string_buffer = StringIO()
    sys.stdout = string_buffer
    try:
        check_estimator(NoSparseClassifier)
    except:
        pass
    finally:
        sys.stdout = old_stdout
    assert_true(msg in string_buffer.getvalue())

    # doesn't error on actual estimator
    check_estimator(LogisticRegression)

def test_check_estimators_unfitted():
    # check that a ValueError/AttributeError is raised when calling predict
    # on an unfitted estimator
    msg = "AttributeError or ValueError not raised by predict"
    assert_raises_regex(AssertionError, msg, check_estimators_unfitted,
                        "estimator", NoSparseClassifier)

    # check that CorrectNotFittedError inherit from either ValueError
    # or AttributeError
    check_estimators_unfitted("estimator", CorrectNotFittedErrorClassifier)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 20 13:37:16 2017
Author: Peiyong Jiang : jiangpeiyong@impcas.ac.cn
Function:
    Check that the Distribution generation method is right.

"""

import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np

from Orth import LambdaR,OrthTrans
from TFOrth import TFLambdaR,TFOrthTrans

plt.close('all')

emitX=4.8
alphaX=-2.3
betaX=15.3
gammaX=(1.+alphaX**2)/betaX

diagRX=LambdaR(emitX,alphaX,betaX,gammaX)

PX=OrthTrans(emitX,alphaX,betaX,gammaX)

numPart=np.int32(1e5)
Z=np.random.randn(2,numPart)

X=np.matmul(np.matmul(PX,np.linalg.inv(diagRX)),Z)

plt.figure(1)
plt.plot(X[0,:],X[1,:],'r.')
plt.axis('equal')

##

def WeightP(shape):
    initial=tf.truncated_normal(shape,stddev=0.1)
    return tf.Variable(initial)

def WeightLambda2D():
    lambda1=tf.Variable(tf.random_uniform([1,1]),dtype=tf.float32)
    lambda2=tf.Variable(tf.random_uniform([1,1]),dtype=tf.float32)

    O=tf.reshape(tf.constant(0,tf.float32),[1,1])

    LambdaR1=tf.concat([lambda1,O],0)
    LambdaR2=tf.concat([O,lambda2],0)
    LambdaR=tf.concat([LambdaR1,LambdaR2],1)

    return LambdaR

P_1=WeightP([2,2])
LambdaR=WeightLambda2D()

xI=tf.placeholder(tf.float32,[2,None])

xL1=tf.matmul(P_1,xI)

xO=tf.matmul(LambdaR,xL1)

xR=xO[0]**2+xO[1]**2

lossXR=(xR-2.)**2

rateLearn=5e-4
optXR=tf.train.AdamOptimizer(rateLearn)

trainXR=optXR.minimize(lossXR)

meanLossXR=tf.reduce_mean(lossXR)

sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))

sess.run(tf.global_variables_initializer())

sizeBatch=64

for _ in xrange(30000):

    startBatch=np.random.randint(0,high=numPart-sizeBatch-1)
    xFeed=X[:,startBatch:startBatch+sizeBatch:]

    sess.run(trainXR,feed_dict={xI:xFeed})

    #print(sess.run(LambdaR))
    #print('---------------------------')
    print(sess.run(meanLossXR,feed_dict={xI:X}))
    print('_______________________________________________')

    '''
    zReal=sess.run(xO,feed_dict={xI:X})

    plt.figure(2)
    plt.clf()
    plt.plot(zReal[0,:],zReal[1,:],'r.')
    plt.axis('equal')
    plt.pause(0.001)
    '''

LambdaRGet=sess.run(LambdaR)

print(LambdaRGet)
print('---------------------------')
print(1./(LambdaRGet[0,0]*LambdaRGet[1,1]))

zReal=sess.run(xO,feed_dict={xI:X})

plt.figure(2)
plt.plot(zReal[0,:],zReal[1,:],'r.')
plt.axis('equal')

'''
print(sess.run(P_1))

print(sess.run(LambdaR))

print(sess.run(xR,feed_dict={xI:X}))
'''

'''
wEmit=tf.Variable([emitX])
wAlpha=tf.Variable([alphaX])
wBeta=tf.Variable([betaX])
wGamma=tf.Variable([gammaX])
'''

'''
wEmit=tf.Variable([13.])
wAlpha=tf.Variable([1.3])
wBeta=tf.Variable([0.5])
#wGamma=tf.Variable([0.5])
wGamma=(1.+wAlpha**2)/wBeta

xH=tf.placeholder(tf.float32,[2,None])

diagR,diagRT=TFLambdaR(wEmit,wAlpha,wBeta,wGamma)
P,PI=TFOrthTrans(wEmit,wAlpha,wBeta,wGamma)

zH=tf.matmul(tf.matmul(diagR,PI),xH)

R=zH[0]**2+zH[1]**2
#lossR=tf.abs(R-2.e-6)
lossR=R

optR=tf.train.GradientDescentOptimizer(0.01)

trainR=optR.minimize(lossR)

sess=tf.Session()

sess.run(tf.global_variables_initializer())
#sess.run(diagR)

print(sess.run(R,feed_dict={xH:X}))

numIter=10
recEmit=np.zeros(numIter)
recAlpha=np.zeros(numIter)
recBeta=np.zeros(numIter)
recGamma=np.zeros(numIter)
recLoss=np.zeros(numIter)

for _ in xrange(numIter):
    sess.run(trainR,feed_dict={xH:X})
    recEmit[_]=sess.run(wEmit)
    recAlpha[_]=sess.run(wAlpha)
    recBeta[_]=sess.run(wBeta)
    recGamma[_]=sess.run(wGamma)
    recLoss[_]=sess.run(tf.reduce_mean(lossR))

print(recEmit)
print(recAlpha)
#print(sess.run(R,feed_dict={xH:X}))

plt.figure('emit')
plt.plot(recEmit)

plt.figure('alpha')
plt.plot(recAlpha)

plt.figure('beta')
plt.plot(recBeta)

plt.figure('gamma')
plt.plot(recGamma)

plt.figure('Loss')
plt.plot(recLoss)

'''

'''
zGet=sess.run(zH,feed_dict={xH:X})
print(sess.run(lossR,feed_dict={xH:X}))
'''
'''
plt.figure('Check')
plt.hold('on')
plt.plot(Z[0,:],Z[1,:],'bo')
plt.plot(zGet[0,:],zGet[1,:],'r.')
plt.axis('equal')
'''

'''
print(sess.run(wEmit))
print(sess.run(wAlpha))
print(sess.run(wBeta))
print(sess.run(wGamma))

print(sess.run(diagR))
print(sess.run(diagRT))
'''
#print(PX)
#print(sess.run(P))
#print(sess.run(zH,feed_dict={xH:X}))


# coding: utf-8
#  import pandas as pd
"""Malloovia interface to LP solver"""

from typing import Sequence, List, Any
from itertools import product as cartesian_product
from inspect import ismethod
from collections import namedtuple
from uuid import uuid4
import os

import pulp  # type: ignore

from pulp import (
    LpContinuous,
    LpInteger,
    LpVariable,
    lpSum,
    LpProblem,
    LpMinimize,
    LpMaximize,
    PulpSolverError,
    COIN_CMD,
    log,
    subprocess,
)

from .solution_model import (
    MallooviaHistogram,
    ReservedAllocation,
    AllocationInfo,
    Status,
    pulp_to_malloovia_status,
)
from .model import System, Workload, App, TimeUnit

LpProblem.bestBound = None  # Add new attribute to pulp problems

class MallooviaLp:
    """Solves the allocation problem, using Linear Programming.

    This class contains methods to create a linear programming problem
    (using PuLP), to add restrictions and extra variables to it,
    to solve it (using PuLP supported solvers), and to retrieve
    the solution in a format amenable to further analysis and display.

    The LP problem instantiates these variables:

    - For reserved instances: ``Y_(_a,_ic)``, where ``Y`` is a fixed prefix,
      ``a`` is a string representation of each application and ``ic`` is the string
      representation of each reserved instance class considered.
      After solving the LP problem, the value of the variable is the number of
      reserved machines of instance class `ic` for application `a`, for the whole
      reservation period.

    - For on-demand instances: ``X_(_a,_ic,_l)``, where ``X`` is a fixed prefix,
      ``a`` is a string representation of each application, ``ic`` is the string
      representation of each on-demand instance class considered and ``l`` is a
      string representation of a "workload tuple", which is a tuple of numbers,
      e.g: ``(1230, 442, 123)``, each one representing the workload of one of the apps.
      After solving the LP problem, the value of the variable is the number of
      on-demand machines of instance class `ic` deployed for application `a` at a
      timeslot which has a workload prediction equal to the tuple ``l``.

    Intended usage:

    1. Instantiate the class (see constructor parameters below).
    2. Call object's ``.create_problem()``.
    3. Call object's ``.solve()``.
    4. Retrieve solution by calling object's ``.get_allocation()`` to get the solution
       for all variables, or ``.get_reserved_allocation()`` to get ony the number of
       reserved instances of each type.
    5. Retrieve the cost of the solution via object's ``.get_solution()``.

    You can use object's property ``pulp_problem`` to access the PuLP problem object
    which represents the linear programming problem, to inspect or save it if required.
    """

    def __init__(
        self,
        system: System,
        workloads: Sequence[Workload],
        preallocation: ReservedAllocation = None,
        relaxed: bool = False,
    ) -> None:
        """Constructor:

        Args:
            system: namedtuple containing "name", "apps", "instance_classes"
                and "performances" for the problem to solve.
            workloads: list of workloads, one per app. Each workload
                is a namedtuple which contains a reference to the app, and a sequence
                of N numbers which is the prediction for the next N timeslots. This
                sequence must have the same length for all workloads in the list.
            preallocation: number of reserved instances which are
                preallocated. In phase I this parameter can be omitted (defaults to ``None``),
                and in phase II it should contain the object returned by
                ``get_reserved_allocation()`` after solving phase I.
            relaxed: if ``True``, the problem uses continuous variables
                instead of integer ones.
        """
        self.system = system
        # Ensure that the workloads received are ordered by the field app in the same
        # ordering than the list system.apps
        self.workloads = reorder_workloads(workloads, system.apps)
        if preallocation is None:
            self.fixed_vms = None
        else:
            assert len(preallocation.instance_classes) == len(
                preallocation.vms_number
            ), (
                "preallocation is wrong, the number of elements in instance_classes and in "
                "vms_number must be the same"
            )
            self.fixed_vms = dict(
                zip(preallocation.instance_classes, preallocation.vms_number)
            )
        self.relaxed = relaxed
        self.pulp_problem: Any = None
        self.load_hist = get_load_hist_from_load(self.workloads)
        self.solver_called = False

        # CookedData  stores some info required when building the problem, so that
        # this data is gathered only once, during __init__, and used when required
        CookedData = namedtuple(  # pylint: disable=invalid-name
            "CookedData",
            [
                "map_dem",
                "map_res",
                "instances_res",
                "instances_dem",
                "limiting_sets",
                "instance_prices",
                "instance_perfs",
            ],
        )

        # Separate the instances in two types: reserved and on-demand
        # Also create dictionaries for fast lookup of price and performance, converted
        # to the timeslot units
        instances_res = []
        instances_dem = []
        instance_prices = {}
        instance_perfs = {}
        timeslot_length = self.workloads[0].time_unit
        for iclass in system.instance_classes:
            instance_prices[iclass] = iclass.price / TimeUnit(iclass.time_unit).to(
                timeslot_length
            )
            for app in self.system.apps:
                instance_perfs[iclass, app] = self.system.performances.values[
                    iclass, app
                ] / TimeUnit(self.system.performances.time_unit).to(timeslot_length)
            if iclass.is_reserved:
                instances_res.append(iclass)
            else:
                instances_dem.append(iclass)

        # Compute the set of LimitingSets (clouds), extracted
        # from the instances
        limiting_sets = set()
        for iclass in system.instance_classes:
            limiting_sets.update(iclass.limiting_sets)

        # Store cooked data
        self.cooked = CookedData(
            map_dem=None,  # To be updated later by create_variables
            map_res=None,
            instances_res=instances_res,
            instances_dem=instances_dem,
            instance_prices=instance_prices,
            instance_perfs=instance_perfs,
            limiting_sets=limiting_sets,
        )

    def _create_variables(self) -> None:
        """Creates the set of variables Y* and X* of the PuLP problem.

        Override it if you need to create extra variables (first use
        ``super().create_variables()`` to call the base class method)."""
        if self.relaxed:
            kind = LpContinuous
        else:
            kind = LpInteger

        # List all combinations of apps and instances and workloads
        comb_res = cartesian_product(self.system.apps, self.cooked.instances_res)
        comb_dem = cartesian_product(
            self.system.apps, self.cooked.instances_dem, self.load_hist.keys()
        )
        map_res = LpVariable.dicts("Y", comb_res, 0, None, kind)
        map_dem = LpVariable.dicts("X", comb_dem, 0, None, kind)
        self.cooked = self.cooked._replace(map_res=map_res, map_dem=map_dem)

    def _cost_function(self) -> None:
        """Adds to the LP problem the function to optimize.

        The function to optimize is the cost of the deployment. It is computed as
        the sum of all Y_a_ic multiplied by the length of the period and by the price/timeslot
        of each reserved instance class plus all X_a_ic_l multiplied by the price/timeslot
        of each on-demand instance class and by the number of times that workload ``l``
        appears in the period."""

        period_length = sum(self.load_hist.values())

        self.pulp_problem += (
            lpSum(
                [
                    self.cooked.map_res[_a, _ic]
                    * self.cooked.instance_prices[_ic]
                    * period_length
                    for _a in self.system.apps
                    for _ic in self.cooked.instances_res
                ]
                + [
                    self.cooked.map_dem[_a, _ic, _l]
                    * self.cooked.instance_prices[_ic]
                    * self.load_hist[_l]
                    for _a in self.system.apps
                    for _ic in self.cooked.instances_dem
                    for _l in self.load_hist.keys()
                ]
            ),
            "Objective: minimize cost",
        )

    def create_problem(self) -> "MallooviaLp":
        """Creates the PuLP problem with all variables and restrictions.

        Returns:
          pulp.LpProblem: instance of the PuLP problem.
        """

        # Create the linear programming problem
        self.pulp_problem = LpProblem(self.system.name, LpMinimize)

        # Once we have the variables represented as tuples, we use
        # the tuples to create the linear programming variables for pulp
        self._create_variables()

        # Create the goal function
        self._cost_function()

        # Add all restrictions indicated with functions *_restriction
        # in this class
        self._add_all_restrictions()

        return self

    def _add_all_restrictions(self) -> None:
        """This functions uses introspection to discover all implemented
        methods whose name ends with ``_restriction``, and runs them all."""
        for name in dir(self):
            attribute = getattr(self, name)
            if ismethod(attribute) and name.endswith("_restriction"):
                attribute()

    def performance_restriction(self) -> None:
        """Adds performance restriction to the problem.

        This restriction forces, for each workload tuple, the performance of the
        solution to be greater than or equal to that workload level for
        all applications.
        """
        for i, app in enumerate(self.system.apps):
            perf_reserved = []
            for ins in self.cooked.instances_res:
                perf_reserved.append(
                    self.cooked.map_res[app, ins] * self.cooked.instance_perfs[ins, app]
                )
            for load in self.load_hist.keys():
                perf_ondemand = []
                for ins in self.cooked.instances_dem:
                    perf_ondemand.append(
                        self.cooked.map_dem[app, ins, load]
                        * self.cooked.instance_perfs[ins, app]
                    )
                self.pulp_problem += (
                    lpSum(perf_reserved + perf_ondemand) >= load[i],
                    "Minimum performance for application {} "
                    "when workload is {}".format(app, load),
                )
        return

    def limit_instances_per_class_restriction(
        self
    ) -> None:  # pylint: disable=invalid-name
        """Adds ``max_vms`` per instance class restriction.

        If the ``ic`` instance has a ``max_vms`` attribute, this is a limit for all
        ``Y_*_ic`` and ``X_*_ic_*`` variables."""
        for ins in self.system.instance_classes:
            if ins.max_vms == 0:
                continue  # No limit for this instance class

            if ins.is_reserved:
                self.pulp_problem += (
                    lpSum(self.cooked.map_res[app, ins] for app in self.system.apps)
                    <= ins.max_vms,
                    "Max instances reserved " "instance class {}".format(ins),
                )
            else:
                for load in self.load_hist.keys():
                    self.pulp_problem += (
                        lpSum(
                            self.cooked.map_dem[app, ins, load]
                            for app in self.system.apps
                        )
                        <= ins.max_vms,
                        "Max instances for on-demand instance "
                        "class {} when workload is {}".format(ins, load),
                    )

    def set_fixed_instances_restriction(self) -> None:
        """Adds restrictions for variables with pre-fixed values.

        For every ``ic`` in ``self.fixed_vms`` a restriction is
        added which forces the total number of those instance classes in
        the solution to be at equal to a given value for reserved instances,
        and at least equal to a given value for on-demand instances.
        This is used mainly in phase II to ensure that reserved instances
        are fixed, or to allow to keep at least some number of on-demand
        instances running from previous timeslots, when using "guided"
        strategies"."""

        if self.fixed_vms is None:  # No fixed instances, we are in PhaseI
            return
        for ins, value in self.fixed_vms.items():
            if ins.is_reserved:
                self.pulp_problem += (
                    lpSum(self.cooked.map_res[app, ins] for app in self.system.apps)
                    == value,
                    "Reserved instance class {} " "is fixed to {}".format(ins, value),
                )
            else:
                for load in self.load_hist.keys():
                    self.pulp_problem += (
                        lpSum(
                            self.cooked.map_dem[app, ins, load]
                            for app in self.system.apps
                        )
                        >= value,
                        "On-demand instance class {} is at least {} "
                        "when workload is {}".format(ins, value, load),
                    )

    def limit_instances_per_limiting_set_restriction(
        self
    ) -> None:  # pylint: disable=invalid-name
        """Adds ``max_vms`` per limiting set restriction.

        If the limiting set provides a max_vms > 0, then the sum of all
        instances which are member of that limiting set should be limited
        to that maximum."""
        for cloud in self.cooked.limiting_sets:
            if cloud.max_vms == 0:
                continue  # No restriction for this limiting set

            for load in self.load_hist.keys():
                self.pulp_problem += (
                    lpSum(
                        [
                            self.cooked.map_res[app, ic]
                            for app in self.system.apps
                            for ic in self.cooked.instances_res
                            if cloud in ic.limiting_sets
                        ]
                        + [
                            self.cooked.map_dem[app, ic, load]
                            for app in self.system.apps
                            for ic in self.cooked.instances_dem
                            if cloud in ic.limiting_sets
                        ]
                    )
                    <= cloud.max_vms,
                    "Max instances for limiting set {} "
                    "when workload is {}".format(cloud, load),
                )

    def limit_cores_per_limiting_set_restriction(
        self
    ) -> None:  # pylint: disable=invalid-name
        """Adds ``max_cores`` per limiting set restriction.

        If the limiting set provides a max_cores > 0, then the sum of all
        instance cores among all instance classes which are member of that
        limiting set should be limited to that maximum."""
        for cloud in self.cooked.limiting_sets:
            if cloud.max_cores == 0:
                continue  # No restriction for this limiting set

            for load in self.load_hist.keys():
                self.pulp_problem += (
                    lpSum(
                        [
                            self.cooked.map_res[app, ic] * ic.cores
                            for app in self.system.apps
                            for ic in self.cooked.instances_res
                            if cloud in ic.limiting_sets
                        ]
                        + [
                            self.cooked.map_dem[app, ic, load] * ic.cores
                            for app in self.system.apps
                            for ic in self.cooked.instances_dem
                            if cloud in ic.limiting_sets
                        ]
                    )
                    <= cloud.max_cores,
                    "Max cores for limiting set {} "
                    "when workload is {}".format(cloud, load),
                )

    def solve(self, *args, **kwargs):
        """Calls PuLP solver.

        Args:
            *args: positional args passed to ``LpProblem.solve()``
            \\**kwargs: keyword args passed to ``LpProblem.solve()``.

        Returns:
            the value returned by ``LpProblem.solve()``.
        """
        self.solver_called = True
        return self.pulp_problem.solve(*args, **kwargs)

    def get_status(self) -> Status:
        """Returns the status of the problem"""
        if not self.solver_called:
            return Status.unsolved
        return pulp_to_malloovia_status(self.pulp_problem.status)

    def get_cost(self) -> float:
        """Gets the cost of the problem, obtained after solving it.

        Returns:
            The cost of the optimal solution found by PuLP.

        Raises:
            ValueError: when the problem is yet unsolved.
        """
        if self.pulp_problem.status != pulp.LpStatusOptimal:
            raise ValueError("Cannot get the cost when the status is not optimal")

        return pulp.value(self.pulp_problem.objective)

    def get_allocation(self) -> AllocationInfo:
        """Retrieves the allocation given by the solution of the LP problem.

        Returns:
            The allocation given by the solution.

        Raises:
            ValueError: if no solution is available (unsolved or infeasible problem)
        """

        if self.pulp_problem.status != pulp.LpStatusOptimal:
            raise ValueError("Cannot get the cost when the status is not optimal")

        workload_tuples = []
        repeats = []
        allocation = []
        for load, repeat in self.load_hist.items():
            workload_tuples.append(load)
            repeats.append(repeat)
            workload_allocation = []
            for app in self.system.apps:
                row = list(
                    self.cooked.map_res[app, i].varValue
                    for i in self.cooked.instances_res
                )
                row.extend(
                    self.cooked.map_dem[app, i, load].varValue
                    for i in self.cooked.instances_dem
                )
                workload_allocation.append(tuple(row))
            allocation.append(tuple(workload_allocation))
        return AllocationInfo(
            apps=tuple(self.system.apps),
            instance_classes=tuple(
                self.cooked.instances_res + self.cooked.instances_dem
            ),
            workload_tuples=workload_tuples,
            repeats=repeats,
            values=tuple(allocation),
            units="vms",
        )

    def get_reserved_allocation(self) -> ReservedAllocation:
        """Retrieves the allocation of reserved instances from the solution of the LP problem.

        Returns:
            The total number of reserved instance classes of each
            type to be purchased for the whole reservation period.
        Raises:
            ValueError: if no solution is available (unsolved or infeasible problem)
        """
        # Returns the solution as a list of numbers, each one
        # representing the required number of vms of each reserved type, stored
        # in the field "vms_number" of the object.

        # This number is valid for any workload tuple, and for every timeslot
        # in the reservation period. Also, it does not depend on the applications
        # because it is the total number of reserved instances for all apps.

        # The returned class also stores the list "instance_classes" which provides
        # the instance class associated with each index in the above table.

        # So, if r is the value returned, the value of r.vms_number[i]
        # (being i an integer) is the number of VMs to be allocated
        # from reserved instance class r.instance_classes[i], for every
        # timeslot and for the set of all apps.

        # This is all the information required for PhaseII.

        if self.pulp_problem.status != pulp.LpStatusOptimal:
            raise ValueError("Cannot get the cost when the status is not optimal")

        allocation: List[float] = []
        for _ in self.load_hist:  # Loop over all possible workloads
            workload_allocation: List[float] = []
            for iclass in self.cooked.instances_res:
                i_allocation = sum(
                    self.cooked.map_res[app, iclass].varValue
                    for app in self.system.apps
                )
                workload_allocation.append(i_allocation)

            # The obtained allocation MUST be the same for any workload
            assert allocation == [] or allocation == workload_allocation
            allocation = workload_allocation

        return ReservedAllocation(
            instance_classes=tuple(self.cooked.instances_res),
            vms_number=tuple(allocation),
        )

class ShortReprTuple(tuple):
    """This class implements a tuple whose repr is not standard
    but uses instead the hash of the tuple, to ensure a constant
    length of the repr.

    This is required to store keys in the histogram, because they
    are used to name LP variables which otherwise would have
    a name too long for the solver if the number of apps is large.
    """
    def __repr__(self):
        return str(hash(self))

def get_load_hist_from_load(workloads: Sequence[Workload]) -> MallooviaHistogram:
    """Computes the histogram of the workloads.

    Args:
        workloads: a sequence of :class:`Workload` objects, each one
            containing the fields ``app`` (which identifies the app producing this
            workload) and ``values`` (which stores a sequence of numbers representing
            the workload for each timeslot for that app).
    Returns:
        A dictionary where the key is the workload for one timeslot,
        expressed as a tuple with one element for each application, and the value
        is the number of timeslots in which that workload was found.
    """

    hist = MallooviaHistogram()
    hist.apps = tuple(w.app for w in workloads)
    timeslots = len(workloads[0].values)
    # Ensure that all workloads have the same length and units
    assert all(
        len(w.values) == timeslots for w in workloads
    ), "All workloads should have the same length"
    # Iterate over tuples of loads, one tuple per timeslot
    workload_tuples = zip(*(w.values for w in workloads))
    for load in workload_tuples:
        hist[ShortReprTuple(load)] += 1
    return hist

def reorder_workloads(
    workloads: Sequence[Workload], apps: Sequence[App]
) -> Sequence[Workload]:
    """Returns the a new workload list ordered as the list of apps.

    Args:
        workloads: Sequence of workloads to reorder
        apps: Sequence of apps which dictate the new ordering

    Returns:
        A new sequence of workloads, ordered by app in the order given by apps argument.
    """
    map_apps_workloads = {workload.app: workload for workload in workloads}
    ordered_workloads = []
    for app in apps:
        ordered_workloads.append(map_apps_workloads[app])
    return tuple(ordered_workloads)

class MallooviaLpMaximizeTimeslotPerformance(MallooviaLp):
    """Find the allocation which maximizes performance for a single timeslot.

    This problem is the dual of MallooviaLp. Instead of minimizing the cost
    while providing the minimum performances, the problem to solve now is
    to maximize the performance without breaking the limits.

    The class inherits from Malloovia the initialization methods as well as
    the ones to get the cost and allocation of the solution, but overrides
    the function to be optimized and some of the constraints.
    """

    def _cost_function(self) -> None:
        """Adds to the LP problem the function to optimize (maximize in this case).

        The function to optimize is the performance of the deployment. However, since
        the system is composed to several applications, no single "performance" exists.
        The solution is to maximize the "fraction of performance fulfilled", i.e., the
        sum of `X(_a,_ic,_l)*_ic.performance/_l[a]` among all `_a` and `_ic`.
        """
        workloads = {wl.app: wl.values[0] for wl in self.workloads}

        self.pulp_problem += (
            lpSum(
                [
                    self.cooked.map_res[_a, _ic]
                    * self.cooked.instance_perfs[_ic, _a]
                    / workloads[_a]
                    for _a in self.system.apps
                    for _ic in self.cooked.instances_res
                ]
                + [
                    self.cooked.map_dem[_a, _ic, _l]
                    * self.cooked.instance_perfs[_ic, _a]
                    / workloads[_a]
                    for _a in self.system.apps
                    for _ic in self.cooked.instances_dem
                    for _l in self.load_hist.keys()
                ]
            ),
            "Objective: maximize fulfilled workload fraction",
        )

    def create_problem(self) -> "MallooviaLpMaximizeTimeslotPerformance":
        """This method creates the PuLP problem, and calls other
        methods to add variables and restrictions to it.
        It initializes the attribute 'self.prob' with the
        instance of the PuLP problem created.
        """

        # Create the linear programming problem
        self.pulp_problem = LpProblem(self.system.name, LpMaximize)

        # Create the linear programming variables for pulp
        self._create_variables()

        # Create the goal function
        self._cost_function()

        # Add all restrictions indicated with functions *_restriction
        # in this class
        self._add_all_restrictions()

        return self

    def performance_restriction(self) -> None:
        """Adds performance restriction to the problem.

        This restriction forces, for each workload tuple, the performance of the
        solution to be less than or equal to that workload level, for
        all applications.
        """
        for i, app in enumerate(self.system.apps):
            perf_reserved = []
            for ins in self.cooked.instances_res:
                perf_reserved.append(
                    self.cooked.map_res[app, ins] * self.cooked.instance_perfs[ins, app]
                )
            for load in self.load_hist.keys():
                perf_ondemand = []
                for ins in self.cooked.instances_dem:
                    perf_ondemand.append(
                        self.cooked.map_dem[app, ins, load]
                        * self.cooked.instance_perfs[ins, app]
                    )
                self.pulp_problem += (
                    lpSum(perf_reserved + perf_ondemand) <= load[i],
                    "Maximum performance for application {} "
                    "when workload is {}".format(app, load),
                )

    def get_cost(self) -> float:
        """Gets the cost of the problem, obtained after solving it.

        Returns:
            The cost of the optimal solution found by PuLP.

        Raises:
            ValueError: when the problem is yet unsolved.
        """
        if self.pulp_problem.status == pulp.LpStatusNotSolved:  # Not solved
            raise ValueError("Cannot get the cost of an unsolved problem")
        return sum(
            self.cooked.instance_prices[ic] * self.cooked.map_res[app, ic].varValue
            for ic in self.cooked.instances_res
            for app in self.system.apps
        ) + sum(
            self.cooked.instance_prices[ic]
            * self.cooked.map_dem[app, ic, wl].varValue
            * self.load_hist[wl]
            for ic in self.cooked.instances_dem
            for app in self.system.apps
            for wl in self.load_hist.keys()
        )

# The following function is used to monkey patch part of PuLP code.
# This modification is aimed to get the value of the optimal best bound
# which is provided by CBC solver as part of the solution, even if
# the solution could not be found due to a time limit
#
# PuLP does not recover this value, but for our analysis is useful
# to estimate the worst-case error of our approximation when the
# exact solution cannot be found in a reasonable time.
#
# The code patches the part in which PuLP calls CBC, so that the standard
# output of CBC is redirected to a logfile. When CBC exits, the code
# inspects the logfile and locates the bestBound value, storing it
# as part of the problem to make it accessible to the python code.
#
# This patch only works when the solver is COIN.

# pylint: disable=invalid-name,too-many-locals,missing-docstring,bare-except,too-many-branches,too-many-statements
def _solve_CBC_patched(self, lp, use_mps=True): # pragma: no cover
    """Solve a MIP problem using CBC, patched from original PuLP function
    to save a log with cbc's output and take from it the best bound."""

    def takeBestBoundFromLog(filename):
        try:
            with open(filename, "r") as f:
                for l in f:
                    if l.startswith("Lower bound:"):
                        return float(l.split(":")[-1])
        except:
            pass
        return None

    if not self.executable(self.path):
        raise PulpSolverError("Pulp: cannot execute %s cwd: %s" %
                              (self.path, os.getcwd()))
    if not self.keepFiles:
        uuid = uuid4().hex
        tmpLp = os.path.join(self.tmpDir, "%s-pulp.lp" % uuid)
        tmpMps = os.path.join(self.tmpDir, "%s-pulp.mps" % uuid)
        tmpSol = os.path.join(self.tmpDir, "%s-pulp.sol" % uuid)
        tmpSol_init = os.path.join(self.tmpDir, "%s-pulp_init.sol" % uuid)
    else:
        tmpLp = lp.name+"-pulp.lp"
        tmpMps = lp.name+"-pulp.mps"
        tmpSol = lp.name+"-pulp.sol"
        tmpSol_init = lp.name + "-pulp_init.sol"
    if use_mps:
        vs, variablesNames, constraintsNames, objectiveName = lp.writeMPS(tmpMps, rename = 1)
        cmds = ' '+tmpMps+" "
        if lp.sense == LpMaximize:
            cmds += 'max '
    else:
        vs = lp.writeLP(tmpLp)
        # In the Lp we do not create new variable or constraint names:
        variablesNames = dict((v.name, v.name) for v in vs)
        constraintsNames = dict((c, c) for c in lp.constraints)
        objectiveName = None
        cmds = ' '+tmpLp+" "
    if self.mip_start:
        self.writesol(tmpSol_init, lp, vs, variablesNames, constraintsNames)
        cmds += 'mips {} '.format(tmpSol_init)
    if self.threads:
        cmds += "threads %s "%self.threads
    if self.fracGap is not None:
        cmds += "ratio %s "%self.fracGap
    if self.maxSeconds is not None:
        cmds += "sec %s "%self.maxSeconds
    if self.presolve:
        cmds += "presolve on "
    if self.strong:
        cmds += "strong %d " % self.strong
    if self.cuts:
        cmds += "gomory on "
        # cbc.write("oddhole on "
        cmds += "knapsack on "
        cmds += "probing on "
    for option in self.options:
        cmds += option+" "
    if self.mip:
        cmds += "branch "
    else:
        cmds += "initialSolve "
    cmds += "printingOptions all "
    cmds += "solution "+tmpSol+" "
    # if self.msg:
    #     pipe = None
    # else:
    #     pipe = open(os.devnull, 'w')
    log.debug(self.path + cmds)
    with open(tmpLp + ".log", 'w') as pipe:
        cbc = subprocess.Popen((self.path + cmds).split(), stdout=pipe,
                               stderr=pipe)
        if cbc.wait() != 0:
            raise PulpSolverError("Pulp: Error while trying to execute " +
                                  self.path)
    if not os.path.exists(tmpSol):
        raise PulpSolverError("Pulp: Error while executing "+self.path)
    if use_mps:
        status, values, reducedCosts, shadowPrices, slacks, sol_status = \
            self.readsol_MPS(tmpSol, lp, lp.variables(), variablesNames, constraintsNames)
    else:
        status, values, reducedCosts, shadowPrices, slacks, sol_status = self.readsol_LP(
            tmpSol, lp, lp.variables()
        )
    lp.assignVarsVals(values)
    lp.assignVarsDj(reducedCosts)
    lp.assignConsPi(shadowPrices)
    lp.assignConsSlack(slacks, activity=True)
    lp.assignStatus(status, sol_status)
    lp.bestBound = takeBestBoundFromLog(tmpLp + ".log")
    if not self.keepFiles:
        for f in [tmpMps, tmpLp, tmpSol, tmpSol_init]:
            try:
                os.remove(f)
            except:
                pass
    return status

# Monkey patching
COIN_CMD.solve_CBC = _solve_CBC_patched

__all__ = [
    "MallooviaLp",
    "get_load_hist_from_load",
    "MallooviaLpMaximizeTimeslotPerformance",
]


# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#
# License: BSD 3 clause

import numpy as np

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import SkipTest
from sklearn.linear_model.bayes import BayesianRidge, ARDRegression
from sklearn import datasets

from sklearn.utils.testing import assert_array_almost_equal

def test_bayesian_on_diabetes():
    # Test BayesianRidge on diabetes
    raise SkipTest("XFailed Test")
    diabetes = datasets.load_diabetes()
    X, y = diabetes.data, diabetes.target

    clf = BayesianRidge(compute_score=True)

    # Test with more samples than features
    clf.fit(X, y)
    # Test that scores are increasing at each iteration
    assert_array_equal(np.diff(clf.scores_) > 0, True)

    # Test with more features than samples
    X = X[:5, :]
    y = y[:5]
    clf.fit(X, y)
    # Test that scores are increasing at each iteration
    assert_array_equal(np.diff(clf.scores_) > 0, True)

def test_toy_bayesian_ridge_object():
    # Test BayesianRidge on toy
    X = np.array([[1], [2], [6], [8], [10]])
    Y = np.array([1, 2, 6, 8, 10])
    clf = BayesianRidge(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)

def test_toy_ard_object():
    # Test BayesianRegression ARD classifier
    X = np.array([[1], [2], [3]])
    Y = np.array([1, 2, 3])
    clf = ARDRegression(compute_score=True)
    clf.fit(X, Y)

    # Check that the model could approximately learn the identity function
    test = [[1], [3], [4]]
    assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)


from datetime import datetime, time, timedelta
from pandas.compat import range
import sys
import os

import nose

import numpy as np

from pandas import Index, DatetimeIndex, Timestamp, Series, date_range, period_range

import pandas.tseries.frequencies as frequencies
from pandas.tseries.tools import to_datetime

import pandas.tseries.offsets as offsets
from pandas.tseries.period import PeriodIndex
import pandas.compat as compat
from pandas.compat import is_platform_windows

import pandas.util.testing as tm
from pandas import Timedelta

def test_to_offset_multiple():
    freqstr = '2h30min'
    freqstr2 = '2h 30min'

    result = frequencies.to_offset(freqstr)
    assert(result == frequencies.to_offset(freqstr2))
    expected = offsets.Minute(150)
    assert(result == expected)

    freqstr = '2h30min15s'
    result = frequencies.to_offset(freqstr)
    expected = offsets.Second(150 * 60 + 15)
    assert(result == expected)

    freqstr = '2h 60min'
    result = frequencies.to_offset(freqstr)
    expected = offsets.Hour(3)
    assert(result == expected)

    freqstr = '15l500u'
    result = frequencies.to_offset(freqstr)
    expected = offsets.Micro(15500)
    assert(result == expected)

    freqstr = '10s75L'
    result = frequencies.to_offset(freqstr)
    expected = offsets.Milli(10075)
    assert(result == expected)

    freqstr = '2800N'
    result = frequencies.to_offset(freqstr)
    expected = offsets.Nano(2800)
    assert(result == expected)

    # malformed
    try:
        frequencies.to_offset('2h20m')
    except ValueError:
        pass
    else:
        assert(False)

def test_to_offset_negative():
    freqstr = '-1S'
    result = frequencies.to_offset(freqstr)
    assert(result.n == -1)

    freqstr = '-5min10s'
    result = frequencies.to_offset(freqstr)
    assert(result.n == -310)

def test_to_offset_leading_zero():
    freqstr = '00H 00T 01S'
    result = frequencies.to_offset(freqstr)
    assert(result.n == 1)

    freqstr = '-00H 03T 14S'
    result = frequencies.to_offset(freqstr)
    assert(result.n == -194)

def test_to_offset_pd_timedelta():
    # Tests for #9064
    td = Timedelta(days=1, seconds=1)
    result = frequencies.to_offset(td)
    expected = offsets.Second(86401)
    assert(expected==result)

    td = Timedelta(days=-1, seconds=1)
    result = frequencies.to_offset(td)
    expected = offsets.Second(-86399)
    assert(expected==result)

    td = Timedelta(hours=1, minutes=10)
    result = frequencies.to_offset(td)
    expected = offsets.Minute(70)
    assert(expected==result)

    td = Timedelta(hours=1, minutes=-10)
    result = frequencies.to_offset(td)
    expected = offsets.Minute(50)
    assert(expected==result)

    td = Timedelta(weeks=1)
    result = frequencies.to_offset(td)
    expected = offsets.Day(7)
    assert(expected==result)

    td1 = Timedelta(hours=1)
    result1 = frequencies.to_offset(td1)
    result2 = frequencies.to_offset('60min')
    assert(result1 == result2)

    td = Timedelta(microseconds=1)
    result = frequencies.to_offset(td)
    expected = offsets.Micro(1)
    assert(expected == result)

    td = Timedelta(microseconds=0)
    tm.assertRaises(ValueError, lambda: frequencies.to_offset(td))

def test_anchored_shortcuts():
    result = frequencies.to_offset('W')
    expected = frequencies.to_offset('W-SUN')
    assert(result == expected)

    result1 = frequencies.to_offset('Q')
    result2 = frequencies.to_offset('Q-DEC')
    expected = offsets.QuarterEnd(startingMonth=12)
    assert(result1 == expected)
    assert(result2 == expected)

    result1 = frequencies.to_offset('Q-MAY')
    expected = offsets.QuarterEnd(startingMonth=5)
    assert(result1 == expected)

def test_get_rule_month():
    result = frequencies._get_rule_month('W')
    assert(result == 'DEC')
    result = frequencies._get_rule_month(offsets.Week())
    assert(result == 'DEC')

    result = frequencies._get_rule_month('D')
    assert(result == 'DEC')
    result = frequencies._get_rule_month(offsets.Day())
    assert(result == 'DEC')

    result = frequencies._get_rule_month('Q')
    assert(result == 'DEC')
    result = frequencies._get_rule_month(offsets.QuarterEnd(startingMonth=12))
    print(result == 'DEC')

    result = frequencies._get_rule_month('Q-JAN')
    assert(result == 'JAN')
    result = frequencies._get_rule_month(offsets.QuarterEnd(startingMonth=1))
    assert(result == 'JAN')

    result = frequencies._get_rule_month('A-DEC')
    assert(result == 'DEC')
    result = frequencies._get_rule_month(offsets.YearEnd())
    assert(result == 'DEC')

    result = frequencies._get_rule_month('A-MAY')
    assert(result == 'MAY')
    result = frequencies._get_rule_month(offsets.YearEnd(month=5))
    assert(result == 'MAY')

class TestFrequencyCode(tm.TestCase):

    def test_freq_code(self):
        self.assertEqual(frequencies.get_freq('A'), 1000)
        self.assertEqual(frequencies.get_freq('3A'), 1000)
        self.assertEqual(frequencies.get_freq('-1A'), 1000)

        self.assertEqual(frequencies.get_freq('W'), 4000)
        self.assertEqual(frequencies.get_freq('W-MON'), 4001)
        self.assertEqual(frequencies.get_freq('W-FRI'), 4005)

        for freqstr, code in compat.iteritems(frequencies._period_code_map):
            result = frequencies.get_freq(freqstr)
            self.assertEqual(result, code)

            result = frequencies.get_freq_group(freqstr)
            self.assertEqual(result, code // 1000 * 1000)

            result = frequencies.get_freq_group(code)
            self.assertEqual(result, code // 1000 * 1000)

    def test_freq_group(self):
        self.assertEqual(frequencies.get_freq_group('A'), 1000)
        self.assertEqual(frequencies.get_freq_group('3A'), 1000)
        self.assertEqual(frequencies.get_freq_group('-1A'), 1000)
        self.assertEqual(frequencies.get_freq_group('A-JAN'), 1000)
        self.assertEqual(frequencies.get_freq_group('A-MAY'), 1000)
        self.assertEqual(frequencies.get_freq_group(offsets.YearEnd()), 1000)
        self.assertEqual(frequencies.get_freq_group(offsets.YearEnd(month=1)), 1000)
        self.assertEqual(frequencies.get_freq_group(offsets.YearEnd(month=5)), 1000)

        self.assertEqual(frequencies.get_freq_group('W'), 4000)
        self.assertEqual(frequencies.get_freq_group('W-MON'), 4000)
        self.assertEqual(frequencies.get_freq_group('W-FRI'), 4000)
        self.assertEqual(frequencies.get_freq_group(offsets.Week()), 4000)
        self.assertEqual(frequencies.get_freq_group(offsets.Week(weekday=1)), 4000)
        self.assertEqual(frequencies.get_freq_group(offsets.Week(weekday=5)), 4000)

    def test_get_to_timestamp_base(self):
        tsb = frequencies.get_to_timestamp_base

        self.assertEqual(tsb(frequencies.get_freq_code('D')[0]),
                         frequencies.get_freq_code('D')[0])
        self.assertEqual(tsb(frequencies.get_freq_code('W')[0]),
                         frequencies.get_freq_code('D')[0])
        self.assertEqual(tsb(frequencies.get_freq_code('M')[0]),
                         frequencies.get_freq_code('D')[0])

        self.assertEqual(tsb(frequencies.get_freq_code('S')[0]),
                         frequencies.get_freq_code('S')[0])
        self.assertEqual(tsb(frequencies.get_freq_code('T')[0]),
                         frequencies.get_freq_code('S')[0])
        self.assertEqual(tsb(frequencies.get_freq_code('H')[0]),
                         frequencies.get_freq_code('S')[0])

    def test_freq_to_reso(self):
        Reso = frequencies.Resolution

        self.assertEqual(Reso.get_str_from_freq('A'), 'year')
        self.assertEqual(Reso.get_str_from_freq('Q'), 'quarter')
        self.assertEqual(Reso.get_str_from_freq('M'), 'month')
        self.assertEqual(Reso.get_str_from_freq('D'), 'day')
        self.assertEqual(Reso.get_str_from_freq('H'), 'hour')
        self.assertEqual(Reso.get_str_from_freq('T'), 'minute')
        self.assertEqual(Reso.get_str_from_freq('S'), 'second')
        self.assertEqual(Reso.get_str_from_freq('L'), 'millisecond')
        self.assertEqual(Reso.get_str_from_freq('U'), 'microsecond')
        self.assertEqual(Reso.get_str_from_freq('N'), 'nanosecond')

        for freq in ['A', 'Q', 'M', 'D', 'H', 'T', 'S', 'L', 'U', 'N']:
            # check roundtrip
            result = Reso.get_freq(Reso.get_str_from_freq(freq))
            self.assertEqual(freq, result)

        for freq in ['D', 'H', 'T', 'S', 'L', 'U']:
            result = Reso.get_freq(Reso.get_str(Reso.get_reso_from_freq(freq)))
            self.assertEqual(freq, result)

    def test_get_freq_code(self):
        # freqstr
        self.assertEqual(frequencies.get_freq_code('A'),
                         (frequencies.get_freq('A'), 1))
        self.assertEqual(frequencies.get_freq_code('3D'),
                         (frequencies.get_freq('D'), 3))
        self.assertEqual(frequencies.get_freq_code('-2M'),
                         (frequencies.get_freq('M'), -2))

        # tuple
        self.assertEqual(frequencies.get_freq_code(('D', 1)),
                         (frequencies.get_freq('D'), 1))
        self.assertEqual(frequencies.get_freq_code(('A', 3)),
                         (frequencies.get_freq('A'), 3))
        self.assertEqual(frequencies.get_freq_code(('M', -2)),
                         (frequencies.get_freq('M'), -2))
        # numeric tuple
        self.assertEqual(frequencies.get_freq_code((1000, 1)), (1000, 1))

        # offsets
        self.assertEqual(frequencies.get_freq_code(offsets.Day()),
                         (frequencies.get_freq('D'), 1))
        self.assertEqual(frequencies.get_freq_code(offsets.Day(3)),
                         (frequencies.get_freq('D'), 3))
        self.assertEqual(frequencies.get_freq_code(offsets.Day(-2)),
                         (frequencies.get_freq('D'), -2))

        self.assertEqual(frequencies.get_freq_code(offsets.MonthEnd()),
                         (frequencies.get_freq('M'), 1))
        self.assertEqual(frequencies.get_freq_code(offsets.MonthEnd(3)),
                         (frequencies.get_freq('M'), 3))
        self.assertEqual(frequencies.get_freq_code(offsets.MonthEnd(-2)),
                         (frequencies.get_freq('M'), -2))

        self.assertEqual(frequencies.get_freq_code(offsets.Week()),
                         (frequencies.get_freq('W'), 1))
        self.assertEqual(frequencies.get_freq_code(offsets.Week(3)),
                         (frequencies.get_freq('W'), 3))
        self.assertEqual(frequencies.get_freq_code(offsets.Week(-2)),
                         (frequencies.get_freq('W'), -2))

        # monday is weekday=0
        self.assertEqual(frequencies.get_freq_code(offsets.Week(weekday=1)),
                         (frequencies.get_freq('W-TUE'), 1))
        self.assertEqual(frequencies.get_freq_code(offsets.Week(3, weekday=0)),
                         (frequencies.get_freq('W-MON'), 3))
        self.assertEqual(frequencies.get_freq_code(offsets.Week(-2, weekday=4)),
                         (frequencies.get_freq('W-FRI'), -2))

_dti = DatetimeIndex

class TestFrequencyInference(tm.TestCase):

    def test_raise_if_period_index(self):
        index = PeriodIndex(start="1/1/1990", periods=20, freq="M")
        self.assertRaises(TypeError, frequencies.infer_freq, index)

    def test_raise_if_too_few(self):
        index = _dti(['12/31/1998', '1/3/1999'])
        self.assertRaises(ValueError, frequencies.infer_freq, index)

    def test_business_daily(self):
        index = _dti(['12/31/1998', '1/3/1999', '1/4/1999'])
        self.assertEqual(frequencies.infer_freq(index), 'B')

    def test_day(self):
        self._check_tick(timedelta(1), 'D')

    def test_day_corner(self):
        index = _dti(['1/1/2000', '1/2/2000', '1/3/2000'])
        self.assertEqual(frequencies.infer_freq(index), 'D')

    def test_non_datetimeindex(self):
        dates = to_datetime(['1/1/2000', '1/2/2000', '1/3/2000'])
        self.assertEqual(frequencies.infer_freq(dates), 'D')

    def test_hour(self):
        self._check_tick(timedelta(hours=1), 'H')

    def test_minute(self):
        self._check_tick(timedelta(minutes=1), 'T')

    def test_second(self):
        self._check_tick(timedelta(seconds=1), 'S')

    def test_millisecond(self):
        self._check_tick(timedelta(microseconds=1000), 'L')

    def test_microsecond(self):
        self._check_tick(timedelta(microseconds=1), 'U')

    def test_nanosecond(self):
        self._check_tick(np.timedelta64(1, 'ns'), 'N')

    def _check_tick(self, base_delta, code):
        b = Timestamp(datetime.now())
        for i in range(1, 5):
            inc = base_delta * i
            index = _dti([b + inc * j for j in range(3)])
            if i > 1:
                exp_freq = '%d%s' % (i, code)
            else:
                exp_freq = code
            self.assertEqual(frequencies.infer_freq(index), exp_freq)

        index = _dti([b + base_delta * 7] +
                     [b + base_delta * j for j in range(3)])
        self.assertIsNone(frequencies.infer_freq(index))

        index = _dti([b + base_delta * j for j in range(3)] +
                     [b + base_delta * 7])

        self.assertIsNone(frequencies.infer_freq(index))

    def test_weekly(self):
        days = ['MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT', 'SUN']

        for day in days:
            self._check_generated_range('1/1/2000', 'W-%s' % day)

    def test_week_of_month(self):
        days = ['MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT', 'SUN']

        for day in days:
            for i in range(1, 5):
                self._check_generated_range('1/1/2000', 'WOM-%d%s' % (i, day))

    def test_fifth_week_of_month(self):
        # Only supports freq up to WOM-4. See #9425
        func = lambda: date_range('2014-01-01', freq='WOM-5MON')
        self.assertRaises(ValueError, func)

    def test_fifth_week_of_month_infer(self):
        # Only attempts to infer up to WOM-4. See #9425
        index = DatetimeIndex(["2014-03-31", "2014-06-30", "2015-03-30"])
        assert frequencies.infer_freq(index) is None

    def test_week_of_month_fake(self):
        #All of these dates are on same day of week and are 4 or 5 weeks apart
        index = DatetimeIndex(["2013-08-27","2013-10-01","2013-10-29","2013-11-26"])
        assert frequencies.infer_freq(index) != 'WOM-4TUE'

    def test_monthly(self):
        self._check_generated_range('1/1/2000', 'M')

    def test_monthly_ambiguous(self):
        rng = _dti(['1/31/2000', '2/29/2000', '3/31/2000'])
        self.assertEqual(rng.inferred_freq, 'M')

    def test_business_monthly(self):
        self._check_generated_range('1/1/2000', 'BM')

    def test_business_start_monthly(self):
        self._check_generated_range('1/1/2000', 'BMS')

    def test_quarterly(self):
        for month in ['JAN', 'FEB', 'MAR']:
            self._check_generated_range('1/1/2000', 'Q-%s' % month)

    def test_annual(self):
        for month in MONTHS:
            self._check_generated_range('1/1/2000', 'A-%s' % month)

    def test_business_annual(self):
        for month in MONTHS:
            self._check_generated_range('1/1/2000', 'BA-%s' % month)

    def test_annual_ambiguous(self):
        rng = _dti(['1/31/2000', '1/31/2001', '1/31/2002'])
        self.assertEqual(rng.inferred_freq, 'A-JAN')

    def _check_generated_range(self, start, freq):
        freq = freq.upper()

        gen = date_range(start, periods=7, freq=freq)
        index = _dti(gen.values)
        if not freq.startswith('Q-'):
            self.assertEqual(frequencies.infer_freq(index), gen.freqstr)
        else:
            inf_freq = frequencies.infer_freq(index)
            self.assertTrue((inf_freq == 'Q-DEC' and
                             gen.freqstr in ('Q', 'Q-DEC', 'Q-SEP', 'Q-JUN',
                                             'Q-MAR'))
                            or
                            (inf_freq == 'Q-NOV' and
                             gen.freqstr in ('Q-NOV', 'Q-AUG', 'Q-MAY', 'Q-FEB'))
                            or
                            (inf_freq == 'Q-OCT' and
                             gen.freqstr in ('Q-OCT', 'Q-JUL', 'Q-APR', 'Q-JAN')))

        gen = date_range(start, periods=5, freq=freq)
        index = _dti(gen.values)
        if not freq.startswith('Q-'):
            self.assertEqual(frequencies.infer_freq(index), gen.freqstr)
        else:
            inf_freq = frequencies.infer_freq(index)
            self.assertTrue((inf_freq == 'Q-DEC' and
                             gen.freqstr in ('Q', 'Q-DEC', 'Q-SEP', 'Q-JUN',
                                             'Q-MAR'))
                            or
                            (inf_freq == 'Q-NOV' and
                             gen.freqstr in ('Q-NOV', 'Q-AUG', 'Q-MAY', 'Q-FEB'))
                            or
                            (inf_freq == 'Q-OCT' and
                             gen.freqstr in ('Q-OCT', 'Q-JUL', 'Q-APR', 'Q-JAN')))

    def test_infer_freq(self):
        rng = period_range('1959Q2', '2009Q3', freq='Q')
        rng = Index(rng.to_timestamp('D', how='e').asobject)
        self.assertEqual(rng.inferred_freq, 'Q-DEC')

        rng = period_range('1959Q2', '2009Q3', freq='Q-NOV')
        rng = Index(rng.to_timestamp('D', how='e').asobject)
        self.assertEqual(rng.inferred_freq, 'Q-NOV')

        rng = period_range('1959Q2', '2009Q3', freq='Q-OCT')
        rng = Index(rng.to_timestamp('D', how='e').asobject)
        self.assertEqual(rng.inferred_freq, 'Q-OCT')

    def test_infer_freq_tz(self):

        freqs = {'AS-JAN': ['2009-01-01', '2010-01-01', '2011-01-01', '2012-01-01'],
                 'Q-OCT': ['2009-01-31', '2009-04-30', '2009-07-31', '2009-10-31'],
                 'M': ['2010-11-30', '2010-12-31', '2011-01-31', '2011-02-28'],
                 'W-SAT': ['2010-12-25', '2011-01-01', '2011-01-08', '2011-01-15'],
                 'D': ['2011-01-01', '2011-01-02', '2011-01-03', '2011-01-04'],
                 'H': ['2011-12-31 22:00', '2011-12-31 23:00', '2012-01-01 00:00', '2012-01-01 01:00']
        }

        # GH 7310
        for tz in [None, 'Australia/Sydney', 'Asia/Tokyo', 'Europe/Paris',
                   'US/Pacific', 'US/Eastern']:
            for expected, dates in compat.iteritems(freqs):
                idx = DatetimeIndex(dates, tz=tz)
                self.assertEqual(idx.inferred_freq, expected)

    def test_infer_freq_tz_transition(self):
        # Tests for #8772
        date_pairs = [['2013-11-02', '2013-11-5'], #Fall DST
                      ['2014-03-08', '2014-03-11'], #Spring DST
                      ['2014-01-01', '2014-01-03']] #Regular Time
        freqs = ['3H', '10T', '3601S', '3600001L', '3600000001U', '3600000000001N']

        for tz in [None, 'Australia/Sydney', 'Asia/Tokyo', 'Europe/Paris',
                   'US/Pacific', 'US/Eastern']:
            for date_pair in date_pairs:
                for freq in freqs:
                    idx = date_range(date_pair[0], date_pair[1], freq=freq, tz=tz)
                    self.assertEqual(idx.inferred_freq, freq)

        index = date_range("2013-11-03", periods=5, freq="3H").tz_localize("America/Chicago")
        self.assertIsNone(index.inferred_freq)

    def test_infer_freq_businesshour(self):
        # GH 7905
        idx = DatetimeIndex(['2014-07-01 09:00', '2014-07-01 10:00', '2014-07-01 11:00',
                             '2014-07-01 12:00', '2014-07-01 13:00', '2014-07-01 14:00'])
        # hourly freq in a day must result in 'H'
        self.assertEqual(idx.inferred_freq, 'H')

        idx = DatetimeIndex(['2014-07-01 09:00', '2014-07-01 10:00', '2014-07-01 11:00',
                             '2014-07-01 12:00', '2014-07-01 13:00', '2014-07-01 14:00',
                             '2014-07-01 15:00', '2014-07-01 16:00',
                             '2014-07-02 09:00', '2014-07-02 10:00', '2014-07-02 11:00'])
        self.assertEqual(idx.inferred_freq, 'BH')

        idx = DatetimeIndex(['2014-07-04 09:00', '2014-07-04 10:00', '2014-07-04 11:00',
                             '2014-07-04 12:00', '2014-07-04 13:00', '2014-07-04 14:00',
                             '2014-07-04 15:00', '2014-07-04 16:00',
                             '2014-07-07 09:00', '2014-07-07 10:00', '2014-07-07 11:00'])
        self.assertEqual(idx.inferred_freq, 'BH')

        idx = DatetimeIndex(['2014-07-04 09:00', '2014-07-04 10:00', '2014-07-04 11:00',
                             '2014-07-04 12:00', '2014-07-04 13:00', '2014-07-04 14:00',
                             '2014-07-04 15:00', '2014-07-04 16:00',
                             '2014-07-07 09:00', '2014-07-07 10:00', '2014-07-07 11:00',
                             '2014-07-07 12:00', '2014-07-07 13:00', '2014-07-07 14:00',
                             '2014-07-07 15:00', '2014-07-07 16:00',
                             '2014-07-08 09:00', '2014-07-08 10:00', '2014-07-08 11:00',
                             '2014-07-08 12:00', '2014-07-08 13:00', '2014-07-08 14:00',
                             '2014-07-08 15:00', '2014-07-08 16:00'])
        self.assertEqual(idx.inferred_freq, 'BH')

    def test_not_monotonic(self):
        rng = _dti(['1/31/2000', '1/31/2001', '1/31/2002'])
        rng = rng[::-1]
        self.assertEqual(rng.inferred_freq, '-1A-JAN')

    def test_non_datetimeindex(self):
        rng = _dti(['1/31/2000', '1/31/2001', '1/31/2002'])

        vals = rng.to_pydatetime()

        result = frequencies.infer_freq(vals)
        self.assertEqual(result, rng.inferred_freq)

    def test_invalid_index_types(self):

        # test all index types
        for i in [ tm.makeIntIndex(10),
                   tm.makeFloatIndex(10),
                   tm.makePeriodIndex(10) ]:
            self.assertRaises(TypeError, lambda : frequencies.infer_freq(i))

        # GH 10822
        # odd error message on conversions to datetime for unicode
        if not is_platform_windows():
            for i in [ tm.makeStringIndex(10),
                       tm.makeUnicodeIndex(10) ]:
                self.assertRaises(ValueError, lambda : frequencies.infer_freq(i))

    def test_string_datetimelike_compat(self):

        # GH 6463
        expected = frequencies.infer_freq(['2004-01', '2004-02', '2004-03', '2004-04'])
        result = frequencies.infer_freq(Index(['2004-01', '2004-02', '2004-03', '2004-04']))
        self.assertEqual(result,expected)

    def test_series(self):

        # GH6407
        # inferring series

        # invalid type of Series
        for s in [ Series(np.arange(10)),
                   Series(np.arange(10.))]:
            self.assertRaises(TypeError, lambda : frequencies.infer_freq(s))

        # a non-convertible string
        self.assertRaises(ValueError, lambda : frequencies.infer_freq(Series(['foo','bar'])))

        # cannot infer on PeriodIndex
        for freq in [None, 'L']:
            s = Series(period_range('2013',periods=10,freq=freq))
            self.assertRaises(TypeError, lambda : frequencies.infer_freq(s))
        for freq in ['Y']:
            with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
                s = Series(period_range('2013',periods=10,freq=freq))
            self.assertRaises(TypeError, lambda : frequencies.infer_freq(s))

        # DateTimeIndex
        for freq in ['M', 'L', 'S']:
            s = Series(date_range('20130101',periods=10,freq=freq))
            inferred = frequencies.infer_freq(s)
            self.assertEqual(inferred,freq)

        s = Series(date_range('20130101','20130110'))
        inferred = frequencies.infer_freq(s)
        self.assertEqual(inferred,'D')

    def test_legacy_offset_warnings(self):
        for k, v in compat.iteritems(frequencies._rule_aliases):
            with tm.assert_produces_warning(FutureWarning):
                result = frequencies.get_offset(k)
            exp = frequencies.get_offset(v)
            self.assertEqual(result, exp)

            with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
                idx = date_range('2011-01-01', periods=5, freq=k)
            exp = date_range('2011-01-01', periods=5, freq=v)
            self.assert_index_equal(idx, exp)

MONTHS = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP',
          'OCT', 'NOV', 'DEC']

def test_is_superperiod_subperiod():
    assert(frequencies.is_superperiod(offsets.YearEnd(), offsets.MonthEnd()))
    assert(frequencies.is_subperiod(offsets.MonthEnd(), offsets.YearEnd()))

    assert(frequencies.is_superperiod(offsets.Hour(), offsets.Minute()))
    assert(frequencies.is_subperiod(offsets.Minute(), offsets.Hour()))

    assert(frequencies.is_superperiod(offsets.Second(), offsets.Milli()))
    assert(frequencies.is_subperiod(offsets.Milli(), offsets.Second()))

    assert(frequencies.is_superperiod(offsets.Milli(), offsets.Micro()))
    assert(frequencies.is_subperiod(offsets.Micro(), offsets.Milli()))

    assert(frequencies.is_superperiod(offsets.Micro(), offsets.Nano()))
    assert(frequencies.is_subperiod(offsets.Nano(), offsets.Micro()))

if __name__ == '__main__':
    import nose
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)


# doc:slow-example
"""
.. _tut_stats_cluster_methods:

======================================================
Permutation t-test on toy data with spatial clustering
======================================================

Following the illustrative example of Ridgway et al. 2012,
this demonstrates some basic ideas behind both the "hat"
variance adjustment method, as well as threshold-free
cluster enhancement (TFCE) methods in mne-python.

This toy dataset consists of a 40 x 40 square with a "signal"
present in the center (at pixel [20, 20]) with white noise
added and a 5-pixel-SD normal smoothing kernel applied.

For more information, see:
Ridgway et al. 2012, "The problem of low variance voxels in
statistical parametric mapping; a new hat avoids a 'haircut'",
NeuroImage. 2012 Feb 1;59(3):2131-41.

Smith and Nichols 2009, "Threshold-free cluster enhancement:
addressing problems of smoothing, threshold dependence, and
localisation in cluster inference", NeuroImage 44 (2009) 83-98.

In the top row plot the T statistic over space, peaking toward the
center. Note that it has peaky edges. Second, with the "hat" variance
correction/regularization, the peak becomes correctly centered. Third,
the TFCE approach also corrects for these edge artifacts. Fourth, the
the two methods combined provide a tighter estimate, for better or
worse.

Now considering multiple-comparisons corrected statistics on these
variables, note that a non-cluster test (e.g., FDR or Bonferroni) would
mis-localize the peak due to sharpness in the T statistic driven by
low-variance pixels toward the edge of the plateau. Standard clustering
(first plot in the second row) identifies the correct region, but the
whole area must be declared significant, so no peak analysis can be done.
Also, the peak is broad. In this method, all significances are
family-wise error rate (FWER) corrected, and the method is
non-parametric so assumptions of Gaussian data distributions (which do
actually hold for this example) don't need to be satisfied. Adding the
"hat" technique tightens the estimate of significant activity (second
plot). The TFCE approach (third plot) allows analyzing each significant
point independently, but still has a broadened estimate. Note that
this is also FWER corrected. Finally, combining the TFCE and "hat"
methods tightens the area declared significant (again FWER corrected),
and allows for evaluation of each point independently instead of as
a single, broad cluster.

Note that this example does quite a bit of processing, so even on a
fast machine it can take a few minutes to complete.
"""
# Authors: Eric Larson <larson.eric.d@gmail.com>
# License: BSD (3-clause)

import numpy as np
from scipy import stats
from functools import partial
import matplotlib.pyplot as plt
# this changes hidden MPL vars:
from mpl_toolkits.mplot3d import Axes3D  # noqa

from mne.stats import (spatio_temporal_cluster_1samp_test,
                       bonferroni_correction, ttest_1samp_no_p)

try:
    from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
    from scikits.learn.feature_extraction.image import grid_to_graph

print(__doc__)

###############################################################################
# Set parameters
# --------------
width = 40
n_subjects = 10
signal_mean = 100
signal_sd = 100
noise_sd = 0.01
gaussian_sd = 5
sigma = 1e-3  # sigma for the "hat" method
threshold = -stats.distributions.t.ppf(0.05, n_subjects - 1)
threshold_tfce = dict(start=0, step=0.2)
n_permutations = 1024  # number of clustering permutations (1024 for exact)

###############################################################################
# Construct simulated data
# ------------------------
#
# Make the connectivity matrix just next-neighbor spatially
n_src = width * width
connectivity = grid_to_graph(width, width)

#    For each "subject", make a smoothed noisy signal with a centered peak
rng = np.random.RandomState(42)
X = noise_sd * rng.randn(n_subjects, width, width)
#    Add a signal at the dead center
X[:, width // 2, width // 2] = signal_mean + rng.randn(n_subjects) * signal_sd
#    Spatially smooth with a 2D Gaussian kernel
size = width // 2 - 1
gaussian = np.exp(-(np.arange(-size, size + 1) ** 2 / float(gaussian_sd ** 2)))
for si in range(X.shape[0]):
    for ri in range(X.shape[1]):
        X[si, ri, :] = np.convolve(X[si, ri, :], gaussian, 'same')
    for ci in range(X.shape[2]):
        X[si, :, ci] = np.convolve(X[si, :, ci], gaussian, 'same')

###############################################################################
# Do some statistics
# ------------------
#
# .. note::
#     X needs to be a multi-dimensional array of shape
#     samples (subjects) x time x space, so we permute dimensions:
X = X.reshape((n_subjects, 1, n_src))

###############################################################################
# Now let's do some clustering using the standard method.
#
# .. note::
#     Not specifying a connectivity matrix implies grid-like connectivity,
#     which we want here:
T_obs, clusters, p_values, H0 = \
    spatio_temporal_cluster_1samp_test(X, n_jobs=1, threshold=threshold,
                                       connectivity=connectivity,
                                       tail=1, n_permutations=n_permutations)

#    Let's put the cluster data in a readable format
ps = np.zeros(width * width)
for cl, p in zip(clusters, p_values):
    ps[cl[1]] = -np.log10(p)
ps = ps.reshape((width, width))
T_obs = T_obs.reshape((width, width))

#     To do a Bonferroni correction on these data is simple:
p = stats.distributions.t.sf(T_obs, n_subjects - 1)
p_bon = -np.log10(bonferroni_correction(p)[1])

#    Now let's do some clustering using the standard method with "hat":
stat_fun = partial(ttest_1samp_no_p, sigma=sigma)
T_obs_hat, clusters, p_values, H0 = \
    spatio_temporal_cluster_1samp_test(X, n_jobs=1, threshold=threshold,
                                       connectivity=connectivity,
                                       tail=1, n_permutations=n_permutations,
                                       stat_fun=stat_fun)

#    Let's put the cluster data in a readable format
ps_hat = np.zeros(width * width)
for cl, p in zip(clusters, p_values):
    ps_hat[cl[1]] = -np.log10(p)
ps_hat = ps_hat.reshape((width, width))
T_obs_hat = T_obs_hat.reshape((width, width))

#    Now the threshold-free cluster enhancement method (TFCE):
T_obs_tfce, clusters, p_values, H0 = \
    spatio_temporal_cluster_1samp_test(X, n_jobs=1, threshold=threshold_tfce,
                                       connectivity=connectivity,
                                       tail=1, n_permutations=n_permutations)
T_obs_tfce = T_obs_tfce.reshape((width, width))
ps_tfce = -np.log10(p_values.reshape((width, width)))

#    Now the TFCE with "hat" variance correction:
T_obs_tfce_hat, clusters, p_values, H0 = \
    spatio_temporal_cluster_1samp_test(X, n_jobs=1, threshold=threshold_tfce,
                                       connectivity=connectivity,
                                       tail=1, n_permutations=n_permutations,
                                       stat_fun=stat_fun)
T_obs_tfce_hat = T_obs_tfce_hat.reshape((width, width))
ps_tfce_hat = -np.log10(p_values.reshape((width, width)))

###############################################################################
# Visualize results
# -----------------
fig = plt.figure(facecolor='w')

x, y = np.mgrid[0:width, 0:width]
kwargs = dict(rstride=1, cstride=1, linewidth=0, cmap='Greens')

Ts = [T_obs, T_obs_hat, T_obs_tfce, T_obs_tfce_hat]
titles = ['T statistic', 'T with "hat"', 'TFCE statistic', 'TFCE w/"hat" stat']
for ii, (t, title) in enumerate(zip(Ts, titles)):
    ax = fig.add_subplot(2, 4, ii + 1, projection='3d')
    ax.plot_surface(x, y, t, **kwargs)
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_title(title)

p_lims = [1.3, -np.log10(1.0 / n_permutations)]
pvals = [ps, ps_hat, ps_tfce, ps_tfce_hat]
titles = ['Standard clustering', 'Clust. w/"hat"',
          'Clust. w/TFCE', 'Clust. w/TFCE+"hat"']
axs = []
for ii, (p, title) in enumerate(zip(pvals, titles)):
    ax = fig.add_subplot(2, 4, 5 + ii)
    plt.imshow(p, cmap='Purples', vmin=p_lims[0], vmax=p_lims[1])
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_title(title)
    axs.append(ax)

plt.tight_layout()
for ax in axs:
    cbar = plt.colorbar(ax=ax, shrink=0.75, orientation='horizontal',
                        fraction=0.1, pad=0.025)
    cbar.set_label('-log10(p)')
    cbar.set_ticks(p_lims)
    cbar.set_ticklabels(['%0.1f' % p for p in p_lims])

plt.show()


#!/usr/bin/env python
# Copyright (C) <2015> EMBL-European Bioinformatics Institute

# This program is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.

# Neither the institution name nor the name roary_plots
# can be used to endorse or promote products derived from
# this software without prior written permission.
# For written permission, please contact <marco@ebi.ac.uk>.

# Products derived from this software may not be called roary_plots
# nor may roary_plots appear in their names without prior written
# permission of the developers. You should have received a copy
# of the GNU General Public License along with this program.
# If not, see <http://www.gnu.org/licenses/>.

__author__ = "Marco Galardini"
__version__ = '0.1.0'

def get_options():
    import argparse

    # create the top-level parser
    description = "Create plots from roary outputs"
    parser = argparse.ArgumentParser(description = description,
                                     prog = 'roary_plots.py')

    parser.add_argument('tree', action='store',
                        help='Newick Tree file', default='accessory_binary_genes.fa.newick')
    parser.add_argument('spreadsheet', action='store',
                        help='Roary gene presence/absence spreadsheet', default='gene_presence_absence.csv')

    parser.add_argument('--version', action='version',
                         version='%(prog)s '+__version__)

    return parser.parse_args()

if __name__ == "__main__":
    options = get_options()

    import matplotlib.pyplot as plt
    import seaborn as sns

    sns.set_style('white')

    import os
    import pandas as pd
    import numpy as np
    from Bio import Phylo

    t = Phylo.read(options.tree, 'newick')

    # Max distance to create better plots
    mdist = max([t.distance(t.root, x) for x in t.get_terminals()])

    # Load roary
    roary = pd.read_table(options.spreadsheet,
                         sep=',',
                         low_memory=False)
    # Set index (group name)
    roary.set_index('Gene', inplace=True)
    # Drop the other info columns
    roary.drop(list(roary.columns[:10]), axis=1, inplace=True)

    # Transform it in a presence/absence matrix (1/0)
    roary.replace('.{2,100}', 1, regex=True, inplace=True)
    roary.replace(np.nan, 0, regex=True, inplace=True)

    # Sort the matrix by the sum of strains presence
    idx = roary.sum(axis=1).order(ascending=False).index
    roary_sorted = roary.ix[idx]

    # Pangenome frequency plot
    plt.figure(figsize=(7, 5))

    plt.hist(roary.sum(axis=1), roary.shape[1],
             histtype="stepfilled", alpha=.7)

    plt.xlabel('Number of genomes')
    plt.ylabel('Number of genes')

    sns.despine(left=True,
                bottom=True)
    plt.savefig('pangenome_frequency.png')
    plt.clf()

    # Sort the matrix according to tip labels in the tree
    roary_sorted = roary_sorted[[x.name for x in t.get_terminals()]]

    # Plot presence/absence matrix against the tree
    with sns.axes_style('whitegrid'):
        fig = plt.figure(figsize=(17, 10))

        ax1=plt.subplot2grid((1,40), (0, 10), colspan=30)
        a=ax1.matshow(roary_sorted.T, cmap=plt.cm.Blues,
                   vmin=0, vmax=1,
                   aspect='auto',
                   interpolation='none',
                    )
        ax1.set_yticks([])
        ax1.set_xticks([])
        ax1.axis('off')

        ax = fig.add_subplot(1,2,1)
        ax=plt.subplot2grid((1,40), (0, 0), colspan=10, axisbg='white')

        fig.subplots_adjust(wspace=0, hspace=0)

        ax1.set_title('Roary matrix\n(%d gene clusters)'%roary.shape[0])

        Phylo.draw(t, axes=ax,
                   show_confidence=False,
                   label_func=lambda x: None,
                   xticks=([],), yticks=([],),
                   ylabel=('',), xlabel=('',),
                   xlim=(-0.01,mdist+0.01),
                   axis=('off',),
                   title=('parSNP tree\n(%d strains)'%roary.shape[1],),
                   do_show=False,
                  )
        plt.savefig('pangenome_matrix.png')
        plt.clf()

    # Plot the pangenome pie chart
    plt.figure(figsize=(10, 10))

    core     = roary[(roary.sum(axis=1) >= roary.shape[1]*0.99) & (roary.sum(axis=1) <= roary.shape[1]     )].shape[0]
    softcore = roary[(roary.sum(axis=1) >= roary.shape[1]*0.95) & (roary.sum(axis=1) <  roary.shape[1]*0.99)].shape[0]
    shell    = roary[(roary.sum(axis=1) >= roary.shape[1]*0.15) & (roary.sum(axis=1) <  roary.shape[1]*0.95)].shape[0]
    cloud    = roary[roary.sum(axis=1)  < roary.shape[1]*0.15].shape[0]

    total = roary.shape[0]

    def my_autopct(pct):
        val=int(round(pct*total/100.0))
        return '{v:d}'.format(v=val)

    a=plt.pie([core, softcore, shell, cloud],
          labels=['core\n(%d <= strains <= %d)'%(roary.shape[1]*.99,roary.shape[1]),
                  'soft-core\n(%d <= strains < %d)'%(roary.shape[1]*.95,roary.shape[1]*.99),
                  'shell\n(%d <= strains < %d)'%(roary.shape[1]*.15,roary.shape[1]*.95),
                  'cloud\n(strains < %d)'%(roary.shape[1]*.15)],
          explode=[0.1, 0.05, 0.02, 0], radius=0.9,
          colors=[(0, 0, 1, float(x)/total) for x in (core, softcore, shell, cloud)],
          autopct=my_autopct)
    plt.savefig('pangenome_pie.png')
    plt.clf()


"""Build a language detector model

The goal of this exercise is to train a linear classifier on text features
that represent sequences of up to 3 consecutive characters so as to be
recognize natural languages by using the frequencies of short character
sequences as 'fingerprints'.

"""
# Author: Olivier Grisel <olivier.grisel@ensta.org>
# License: Simplified BSD

import sys

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import Perceptron
from sklearn.pipeline import Pipeline
from sklearn.datasets import load_files
from sklearn.cross_validation import train_test_split
from sklearn import metrics

# The training data folder must be passed as first argument
languages_data_folder = sys.argv[1]
dataset = load_files(languages_data_folder)

# Split the dataset in training and test set:
docs_train, docs_test, y_train, y_test = train_test_split(
    dataset.data, dataset.target, test_size=0.5)

# TASK: Build a an vectorizer that splits strings into sequence of 1 to 3
# characters instead of word tokens

# TASK: Build a vectorizer / classifier pipeline using the previous analyzer
# the pipeline instance should stored in a variable named clf

# TASK: Fit the pipeline on the training set

# TASK: Predict the outcome on the testing set in a variable named y_predicted

# Print the classification report
print(metrics.classification_report(y_test, y_predicted,
                                    target_names=dataset.target_names))

# Plot the confusion matrix
cm = metrics.confusion_matrix(y_test, y_predicted)
print(cm)

#import pylab as pl
#pl.matshow(cm, cmap=pl.cm.jet)
#pl.show()

# Predict the result on some short new sentences:
sentences = [
    u'This is a language detection test.',
    u'Ceci est un test de d\xe9tection de la langue.',
    u'Dies ist ein Test, um die Sprache zu erkennen.',
]
predicted = clf.predict(sentences)

for s, p in zip(sentences, predicted):
    print(u'The language of "%s" is "%s"' % (s, dataset.target_names[p]))