"""Class to perform under-sampling by generating centroids based on
clustering."""

# Authors: Guillaume Lemaitre <g.lemaitre58@gmail.com>
#          Fernando Nogueira
#          Christos Aridas
# License: MIT

from __future__ import division, print_function

import numpy as np
from scipy import sparse

from sklearn.cluster import KMeans
from sklearn.neighbors import NearestNeighbors
from sklearn.utils import safe_indexing

from ..base import BaseUnderSampler

VOTING_KIND = ('auto', 'hard', 'soft')

class ClusterCentroids(BaseUnderSampler):
    """Perform under-sampling by generating centroids based on
    clustering methods.

    Method that under samples the majority class by replacing a
    cluster of majority samples by the cluster centroid of a KMeans
    algorithm.  This algorithm keeps N majority samples by fitting the
    KMeans algorithm with N cluster to the majority class and using
    the coordinates of the N cluster centroids as the new majority
    samples.

    Read more in the :ref:`User Guide <cluster_centroids>`.

    Parameters
    ----------
    ratio : str, dict, or callable, optional (default='auto')
        Ratio to use for resampling the data set.

        - If ``str``, has to be one of: (i) ``'minority'``: resample the
          minority class; (ii) ``'majority'``: resample the majority class,
          (iii) ``'not minority'``: resample all classes apart of the minority
          class, (iv) ``'all'``: resample all classes, and (v) ``'auto'``:
          correspond to ``'all'`` with for over-sampling methods and ``'not
          minority'`` for under-sampling methods. The classes targeted will be
          over-sampled or under-sampled to achieve an equal number of sample
          with the majority or minority class.
        - If ``dict``, the keys correspond to the targeted classes. The values
          correspond to the desired number of samples.
        - If callable, function taking ``y`` and returns a ``dict``. The keys
          correspond to the targeted classes. The values correspond to the
          desired number of samples.

    random_state : int, RandomState instance or None, optional (default=None)
        If int, ``random_state`` is the seed used by the random number
        generator; If ``RandomState`` instance, random_state is the random
        number generator; If ``None``, the random number generator is the
        ``RandomState`` instance used by ``np.random``.

    estimator : object, optional(default=KMeans())
        Pass a :class:`sklearn.cluster.KMeans` estimator.

    voting : str, optional (default='auto')
        Voting strategy to generate the new samples:

        - If ``'hard'``, the nearest-neighbors of the centroids found using the
          clustering algorithm will be used.
        - If ``'soft'``, the centroids found by the clustering algorithm will
          be used.
        - If ``'auto'``, if the input is sparse, it will default on ``'hard'``
          otherwise, ``'soft'`` will be used.

        .. versionadded:: 0.3.0

    n_jobs : int, optional (default=1)
        The number of threads to open if possible.

    Notes
    -----
    Supports mutli-class resampling by sampling each class independently.

    See :ref:`sphx_glr_auto_examples_under-sampling_plot_cluster_centroids.py`.

    Examples
    --------

    >>> from collections import Counter
    >>> from sklearn.datasets import make_classification
    >>> from imblearn.under_sampling import \
ClusterCentroids # doctest: +NORMALIZE_WHITESPACE
    >>> X, y = make_classification(n_classes=2, class_sep=2,
    ... weights=[0.1, 0.9], n_informative=3, n_redundant=1, flip_y=0,
    ... n_features=20, n_clusters_per_class=1, n_samples=1000, random_state=10)
    >>> print('Original dataset shape {}'.format(Counter(y)))
    Original dataset shape Counter({1: 900, 0: 100})
    >>> cc = ClusterCentroids(random_state=42)
    >>> X_res, y_res = cc.fit_sample(X, y)
    >>> print('Resampled dataset shape {}'.format(Counter(y_res)))
    ... # doctest: +ELLIPSIS
    Resampled dataset shape Counter({...})

    """

    def __init__(self,
                 ratio='auto',
                 random_state=None,
                 estimator=None,
                 voting='auto',
                 n_jobs=1):
        super(ClusterCentroids, self).__init__(
            ratio=ratio)
        self.random_state = random_state
        self.estimator = estimator
        self.voting = voting
        self.n_jobs = n_jobs

    def _validate_estimator(self):
        """Private function to create the KMeans estimator"""
        if self.estimator is None:
            self.estimator_ = KMeans(
                random_state=self.random_state, n_jobs=self.n_jobs)
        elif isinstance(self.estimator, KMeans):
            self.estimator_ = self.estimator
        else:
            raise ValueError('`estimator` has to be a KMeans clustering.'
                             ' Got {} instead.'.format(type(self.estimator)))

    def _generate_sample(self, X, y, centroids, target_class):
        if self.voting_ == 'hard':
            nearest_neighbors = NearestNeighbors(n_neighbors=1)
            nearest_neighbors.fit(X, y)
            indices = nearest_neighbors.kneighbors(centroids,
                                                   return_distance=False)
            X_new = safe_indexing(X, np.squeeze(indices))
        else:
            if sparse.issparse(X):
                X_new = sparse.csr_matrix(centroids)
            else:
                X_new = centroids
        y_new = np.array([target_class] * centroids.shape[0])

        return X_new, y_new

    def _sample(self, X, y):
        """Resample the dataset.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Matrix containing the data which have to be sampled.

        y : array-like, shape (n_samples,)
            Corresponding label for each sample in X.

        Returns
        -------
        X_resampled : {ndarray, sparse matrix}, shape \
(n_samples_new, n_features)
            The array containing the resampled data.

        y_resampled : ndarray, shape (n_samples_new,)
            The corresponding label of `X_resampled`

        """
        self._validate_estimator()

        if self.voting == 'auto':
            if sparse.issparse(X):
                self.voting_ = 'hard'
            else:
                self.voting_ = 'soft'
        else:
            if self.voting in VOTING_KIND:
                self.voting_ = self.voting
            else:
                raise ValueError("'voting' needs to be one of {}. Got {}"
                                 " instead.".format(VOTING_KIND, self.voting))

        X_resampled, y_resampled = [], []
        for target_class in np.unique(y):
            if target_class in self.ratio_.keys():
                n_samples = self.ratio_[target_class]
                self.estimator_.set_params(**{'n_clusters': n_samples})
                self.estimator_.fit(X[y == target_class])
                X_new, y_new = self._generate_sample(
                    X, y, self.estimator_.cluster_centers_, target_class)
                X_resampled.append(X_new)
                y_resampled.append(y_new)
            else:
                target_class_indices = np.flatnonzero(y == target_class)
                X_resampled.append(safe_indexing(X, target_class_indices))
                y_resampled.append(safe_indexing(y, target_class_indices))

        if sparse.issparse(X):
            X_resampled = sparse.vstack(X_resampled)
        else:
            X_resampled = np.vstack(X_resampled)
        y_resampled = np.hstack(y_resampled)

        return X_resampled, np.array(y_resampled)


"""
Pynab nzedb pre import

Imports pre files from nzedb dropbox

Usage:
    nzedb_pre_import.py large|small

Options:
    -h --help       Show this screen.
    --version       Show version.

"""
# This is quite possibly the most hilariously complex import process...
# What I can gather as the column names from the csv, in case anyone else wants to do this.
# title 1, nfo, size, files, filename 9, nuked 11, nukereason, category 15 , predate 17, source 19, requestid 21, groupname 23

import os
import sys

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))

from pynab.db import db_session, engine, Pre, copy_file
from pynab import releases
import urllib
import regex
import json
import io
from docopt import docopt
from pySmartDL import SmartDL

# Panadas is required
try:
    import pandas
except:
    print("pandas is required to use nzedb pre import: pip install pandas")

# BeautifulSoup is required
try:
    from bs4 import BeautifulSoup
except:
    print("BeautifulSoup is required to use nzedb pre import: pip install beautifulsoup4")

# Regex used to strip out the file name
FILENAME_REGEX = regex.compile(
    "https:\/\/raw.githubusercontent.com\/nZEDb\/nZEDbPre_Dumps\/master\/dumps\/(?P<lastfile>.+)_.+_.+")
COLNAMES = ["name", "filename", "nuked", "category", "pretime", "source", "requestid", "requestgroup"]
INSERTFAILS = []

def nzedbPre():
    downloadLinks = []
    try:
        rawpreJSON = urllib.request.urlopen("https://api.github.com/repositories/45781004/contents/dumps").read()
    except:
        print("pre-import: Error connecting to dropbox, try again later")

    try:
        data = open('lastfile.json')
        lastFileFromDisk = json.load(data)
    except:
        print("pre-import: No existinfg file found, will attempt to download and insert all pres")
        lastFileFromDisk = None

    preJSON = json.loads(rawpreJSON.decode('utf8'))

    for x in preJSON:
        if x["name"] != "0README.txt":
            downloadLinks.append(x["download_url"])

    # Try and process each of the csv's. If they are
    for preCSV in downloadLinks:
        processingFile = FILENAME_REGEX.search(preCSV).groupdict()

        if lastFileFromDisk is None or int(processingFile['lastfile']) > lastFileFromDisk['lastfile']:

            try:
                print("pre-import: Attempting to download file: {}".format(processingFile['lastfile']))
                urllib.request.urlretrieve(preCSV, "unformattedDL.gz")
            except:
                print("pre-import: Error downloading: {} - Please run the process again".format(preCSV))
                INSERTFAILS.append(processingFile['lastfile'])
                # The assumption here is, if one fails, you should probably just start again at that file.
                break

            # Get the data into datatable, much easier to work with.
            dirtyFile = pandas.read_csv('unformattedDL.gz', sep='\t', compression='gzip', header=None, na_values='\\N',
                                        usecols=[0, 8, 10, 14, 16, 18, 20, 22], names=COLNAMES)

            # Clean and process the file
            process(dirtyFile, processingFile)

        else:
            print("pre-import: More than likely {} has already been imported".format(processingFile['lastfile']))
            pass

    if INSERTFAILS is not None:
        print("pre-import: Failures: {}".format(INSERTFAILS))

def largeNzedbPre():
    if os.path.isfile('predb_dump-062714.csv.gz'):
        fileExists = True
    else:
        try:
            url = "https://www.dropbox.com/s/btr42dtzzyu3hh3/predb_dump-062714.csv.gz?dl=1"
            dest = "."

            print("pre-import: File predb_dump-062714.csv not found, attempt to download - may take a while, its 300mb")

            obj = SmartDL(url, dest)
            obj.start()

            fileExists = True
        except:
            print("pre-import: Error downloading/unzipping. Please try again.")
            exit(0)

    if fileExists:
        dirtyChunk = pandas.read_table('predb_dump-062714.csv.gz', compression='gzip', sep='\t', header=None,
                                       na_values='\\N', usecols=[0, 8, 10, 14, 16, 18, 20, 22], names=COLNAMES,
                                       chunksize=10000, engine='c', error_bad_lines=False, warn_bad_lines=False)
    else:
        print("pre-import: File predb_dump-062714.csv not found, please try again.")
        exit(0)

    i = 0
    for chunk in dirtyChunk:
        process(chunk)
        print("pre-import: Imported chunk {}".format(i))
        i += 1

def process(precsv, processingFile=None):
    ordering = ['name', 'filename', 'nuked', 'category', 'pretime', 'source', 'requestid', 'requestgroup', 'searchname']

    # Clean up the file a bit.
    precsv.replace("'", "", inplace=True, regex=True)
    precsv["nuked"].replace("2", "0", inplace=True)
    precsv["nuked"].replace("3", "1", inplace=True)
    precsv["nuked"].replace("4", "1", inplace=True)
    precsv["nuked"].replace("5", "1", inplace=True)
    precsv["nuked"].replace("69", "0", inplace=True)
    precsv.replace(".\\N$", '', inplace=True, regex=True)

    # Sometimes there are duplicates within the table itself, remove them
    precsv.drop_duplicates(subset='name', take_last=True, inplace=True)

    # Add clean searchname column
    precsv['searchname'] = precsv['name'].map(lambda name: releases.clean_release_name(name))

    # Drop the pres without requestid's
    precsv = precsv[precsv.requestid != '0']

    # Create a list of names to check if they exist
    names = list(precsv.name)

    # Query to find any existing pres, we need to delete them so COPY doesn't fail
    prenamelist = []
    with db_session() as db:

        if names:
            pres = db.query(Pre).filter(Pre.name.in_(names)).all()

            for pre in pres:
                prenamelist.append(pre.name)

        data = io.StringIO()
        precsv.to_csv(data, index=False, header=False)

        # Delete any pres found as we are essentially going to update them
        if prenamelist:
            for pre in pres:
                db.delete(pre)
            db.commit()
            print("pre-import: Deleted {} pres that will re-inserted".format(len(prenamelist)))
        else:
            print("pre-import: File clean, no pres need to be deleted before re-insert")

    try:
        if processingFile is not None:
            print("pre-import: Attempting to add {} to the database".format(processingFile['lastfile']))

            data.seek(0)
            copy_file(engine, data, ordering, Pre)

            # Write out the last pre csv name so it can be restarted later without downloading all the pres.
            with open('lastfile.json', 'w') as outfile:
                json.dump({'lastfile': int(processingFile['lastfile'])}, outfile)

        else:
            data.seek(0)
            copy_file(engine, data, ordering, Pre)
            data.close()
            print("pre-import: Chunk import successful")

    except Exception as e:
        print("pre-import: Error inserting into database - {}".format(e))

        if processingFile is not None:
            INSERTFAILS.append(processingFile['lastfile'])
        else:
            print("pre-import: Error processing chunk")

if __name__ == '__main__':

    arguments = docopt(__doc__)

    if arguments['small']:
        nzedbPre()
    elif arguments['large']:
        largeNzedbPre()


from __future__ import print_function, division
import matplotlib
import logging
from sys import stdout
matplotlib.use('Agg') # Must be before importing matplotlib.pyplot or pylab!
from neuralnilm import (Net, RealApplianceSource,
                        BLSTMLayer, DimshuffleLayer,
                        BidirectionalRecurrentLayer)
from neuralnilm.source import standardise, discretize, fdiff, power_and_fdiff
from neuralnilm.experiment import run_experiment, init_experiment
from neuralnilm.net import TrainingError
from neuralnilm.layers import MixtureDensityLayer
from neuralnilm.objectives import scaled_cost, mdn_nll
from neuralnilm.plot import MDNPlotter

from lasagne.nonlinearities import sigmoid, rectify, tanh
from lasagne.objectives import mse
from lasagne.init import Uniform, Normal
from lasagne.layers import (LSTMLayer, DenseLayer, Conv1DLayer,
                            ReshapeLayer, FeaturePoolLayer, RecurrentLayer)
from lasagne.updates import nesterov_momentum, momentum
from functools import partial
import os
import __main__
from copy import deepcopy
from math import sqrt
import numpy as np
import theano.tensor as T

NAME = os.path.splitext(os.path.split(__main__.__file__)[1])[0]
PATH = "/homes/dk3810/workspace/python/neuralnilm/figures"
SAVE_PLOT_INTERVAL = 500
GRADIENT_STEPS = 100

source_dict = dict(
    filename='/data/dk3810/ukdale.h5',
    appliances=[
        ['fridge freezer', 'fridge', 'freezer'],
        'hair straighteners',
        'television'
        # 'dish washer',
        # ['washer dryer', 'washing machine']
    ],
    max_appliance_powers=[300, 500, 200, 2500, 2400],
    on_power_thresholds=[5] * 5,
#    max_input_power=5900,
    min_on_durations=[60, 60, 60, 1800, 1800],
    min_off_durations=[12, 12, 12, 1800, 600],
    window=("2013-06-01", "2014-07-01"),
    seq_length=512,
    output_one_appliance=False,
    boolean_targets=False,
    train_buildings=[1],
    validation_buildings=[1],
#    skip_probability=0.7,
    n_seq_per_batch=16,
    subsample_target=4,
    include_diff=False,
    clip_appliance_power=True,
    target_is_prediction=False,
    standardise_input=True,
    standardise_targets=True,
    input_padding=0,
    lag=0,
    reshape_target_to_2D=True
    # input_stats={'mean': np.array([ 0.05526326], dtype=np.float32),
    #              'std': np.array([ 0.12636775], dtype=np.float32)},
    # target_stats={
    #     'mean': np.array([ 0.04066789,  0.01881946,
    #                        0.24639061,  0.17608672,  0.10273963],
    #                      dtype=np.float32),
    #     'std': np.array([ 0.11449792,  0.07338708,
    #                    0.26608968,  0.33463112,  0.21250485],
    #                  dtype=np.float32)}
)

net_dict = dict(
    save_plot_interval=SAVE_PLOT_INTERVAL,
    loss_function=lambda x, t: mdn_nll(x, t).mean(),
#    loss_function=lambda x, t: mse(x, t).mean(),
    updates_func=momentum,
    learning_rate=1e-03,
   learning_rate_changes_by_iteration={
       100: 5e-04,
       500: 1e-04,
       4000: 5e-05,
       8000: 1e-05
        # 3000: 5e-06,
        # 4000: 1e-06,
        # 10000: 5e-07,
        # 50000: 1e-07
    },
    plotter=MDNPlotter
)

def exp_a(name):
    global source
    source_dict_copy = deepcopy(source_dict)
    source = RealApplianceSource(**source_dict_copy)
    net_dict_copy = deepcopy(net_dict)
    net_dict_copy.update(dict(
        experiment_name=name,
        source=source
    ))
    N = 50
    net_dict_copy['layers_config'] = [
        {
            'type': BidirectionalRecurrentLayer,
            'num_units': N,
            'gradient_steps': GRADIENT_STEPS,
            'W_in_to_hid': Normal(std=1.),
            'nonlinearity': tanh
        },
        {
            'type': FeaturePoolLayer,
            'ds': 4, # number of feature maps to be pooled together
            'axis': 1, # pool over the time axis
            'pool_function': T.max
        },
        {
            'type': BidirectionalRecurrentLayer,
            'num_units': N,
            'gradient_steps': GRADIENT_STEPS,
            'W_in_to_hid': Normal(std=1/sqrt(N)),
            'nonlinearity': tanh
        },
        {
            'type': MixtureDensityLayer,
            'num_units': source.n_outputs,
            'num_components': 2
        }
    ]
    net = Net(**net_dict_copy)
    return net

def main():
    #     EXPERIMENTS = list('abcdefghijklmnopqrstuvwxyz')
    EXPERIMENTS = list('a')
    for experiment in EXPERIMENTS:
        full_exp_name = NAME + experiment
        func_call = init_experiment(PATH, experiment, full_exp_name)
        logger = logging.getLogger(full_exp_name)
        try:
            net = eval(func_call)
            run_experiment(net, epochs=None)
        except KeyboardInterrupt:
            logger.info("KeyboardInterrupt")
            break
        except Exception as exception:
            logger.exception("Exception")
            raise
        finally:
            logging.shutdown()

if __name__ == "__main__":
    main()


import os
import pandas as pd
from Analysis import chckMake
np = pd.np

class MatplotlibPlotter(object):
    def __init__(self,path, analysis):
        self.path = path
        self.analysis = analysis

    def plot_multiple(self, windSpeedCol, powerCol, meanPowerCurveObj):
        try:
            from matplotlib import pyplot as plt
            plt.ioff()
            plotTitle = "Power Curve"

            meanPowerCurve = meanPowerCurveObj.powerCurveLevels[[windSpeedCol,powerCol,'Data Count']][meanPowerCurveObj.powerCurveLevels['Data Count'] > 0 ].reset_index().set_index(windSpeedCol)
            ax = meanPowerCurve[powerCol].plot(color='#00FF00',alpha=0.95,linestyle='--',label='Mean Power Curve')

            colourmap = plt.cm.gist_ncar
            colours = [colourmap(i) for i in np.linspace(0, 0.9, len(self.analysis.dataFrame[self.analysis.nameColumn].unique()))]

            for i,name in enumerate(self.analysis.dataFrame[self.analysis.nameColumn].unique()):
                ax = self.analysis.dataFrame[self.analysis.dataFrame[self.analysis.nameColumn] == name].plot(ax = ax, kind='scatter', x=windSpeedCol, y=powerCol, title=plotTitle, alpha=0.2, label=name, color = colours[i])

            ax.legend(loc=4, scatterpoints = 1)
            ax.set_xlim([min(self.analysis.dataFrame[windSpeedCol].min(),meanPowerCurve.index.min()), max(self.analysis.dataFrame[windSpeedCol].max(),meanPowerCurve.index.max()+2.0)])
            ax.set_xlabel(windSpeedCol + ' (m/s)')
            ax.set_ylabel(powerCol + ' (kW)')
            file_out = self.path + "/Multiple Dataset PowerCurve - " + powerCol + " vs " + windSpeedCol + ".png"
            chckMake(self.path)
            plt.savefig(file_out)
            plt.close()
            return file_out
        except:
            print "Tried to make a power curve scatter chart for multiple data source (%s). Couldn't." % meanPowerCurveObj.name

    def plotPowerCurveSensitivityVariationMetrics(self):
        try:
            from matplotlib import pyplot as plt
            plt.ioff()
            (self.analysis.powerCurveSensitivityVariationMetrics*100.).plot(kind = 'bar', title = 'Summary of Power Curve Variation by Variable. Significance Threshold = %.2f%%' % (self.analysis.sensitivityAnalysisThreshold * 100), figsize = (12,8))
            plt.ylabel('Variation Metric (%)')
            file_out = self.path + os.sep + 'Power Curve Sensitivity Analysis Variation Metric Summary.png'
            plt.savefig(file_out)
            plt.close('all')
        except:
            print "Tried to plot summary of Power Curve Sensitivity Analysis Variation Metric. Couldn't."
        self.analysis.powerCurveSensitivityVariationMetrics.to_csv(self.path + os.sep + 'Power Curve Sensitivity Analysis Variation Metric.csv')

    def plotPowerCurveSensitivity(self, sensCol):
        try:
            df = self.analysis.powerCurveSensitivityResults[sensCol].reset_index()
            from matplotlib import pyplot as plt
            plt.ioff()
            fig = plt.figure(figsize = (12,5))
            fig.suptitle('Power Curve Sensitivity to %s' % sensCol)
            ax1 = fig.add_subplot(121)
            ax1.hold(True)
            ax2 = fig.add_subplot(122)
            ax2.hold(True)
            power_column = self.analysis.measuredTurbulencePower if self.analysis.turbRenormActive else self.analysis.actualPower
            for label in self.analysis.sensitivityLabels.keys():
                filt = df['Bin'] == label
                ax1.plot(df['Wind Speed Bin'][filt], df[power_column][filt], label = label, color = self.analysis.sensitivityLabels[label])
                ax2.plot(df['Wind Speed Bin'][filt], df['Energy Delta MWh'][filt], label = label, color = self.analysis.sensitivityLabels[label])
            ax1.set_xlabel('Wind Speed (m/s)')
            ax1.set_ylabel('Power (kW)')
            ax2.set_xlabel('Wind Speed (m/s)')
            ax2.set_ylabel('Energy Difference from Mean (MWh)')
            box1 = ax1.get_position()
            box2 = ax2.get_position()
            ax1.set_position([box1.x0 - 0.05 * box1.width, box1.y0 + box1.height * 0.17,
                         box1.width * 0.95, box1.height * 0.8])
            ax2.set_position([box2.x0 + 0.05 * box2.width, box2.y0 + box2.height * 0.17,
                         box2.width * 1.05, box2.height * 0.8])
            handles, labels = ax1.get_legend_handles_labels()
            fig.legend(handles, labels, loc='lower center', ncol = len(self.analysis.sensitivityLabels.keys()), fancybox = True, shadow = True)
            file_out = self.path + os.sep + 'Power Curve Sensitivity to %s.png' % sensCol
            chckMake(self.path)
            fig.savefig(file_out)
            plt.close()
        except:
            print "Tried to make a plot of power curve sensitivity to %s. Couldn't." % sensCol

    def plotBy(self,by,variable,df):
        import turbine
        if not isinstance(df,turbine.PowerCurve):
            kind = 'scatter'
        else:
            kind = 'line'
            df=df.powerCurveLevels[df.powerCurveLevels['Input Hub Wind Speed'] <= self.analysis.allMeasuredPowerCurve.cutOutWindSpeed]
        try:
            from matplotlib import pyplot as plt
            plt.ioff()
            ax = df.plot(kind=kind,x=by ,y=variable,title=variable+" By " +by,alpha=0.6,legend=None)
            ax.set_xlim([df[by].min()-1,df[by].max()+1])
            ax.set_xlabel(by)
            ax.set_ylabel(variable)
            file_out = self.path + "/"+variable.replace(" ","_")+"_By_"+by.replace(" ","_")+".png"
            chckMake(self.path)
            plt.savefig(file_out)
            plt.close()
            return file_out
        except:
            print "Tried to make a " + variable.replace(" ","_") + "_By_"+by.replace(" ","_")+" chart. Couldn't."

    def plotPowerCurve(self, windSpeedCol, powerCol, meanPowerCurveObj, anon = False, row_filt = None, fname = None, show_analysis_pc = True, mean_title = 'Mean Power Curve', mean_pc_color = '#00FF00'):
        try:
            from matplotlib import pyplot as plt
            plt.ioff()
            df = self.analysis.dataFrame.loc[row_filt, :] if row_filt is not None else self.analysis.dataFrame
            if (windSpeedCol == self.analysis.densityCorrectedHubWindSpeed) or ((windSpeedCol == self.analysis.inputHubWindSpeed) and (self.analysis.densityCorrectionActive)):
                plotTitle = "Power Curve (corrected to {dens} kg/m^3)".format(dens=self.analysis.referenceDensity)
            else:
                plotTitle = "Power Curve"
            ax = df.plot(kind='scatter', x=windSpeedCol, y=powerCol, title=plotTitle, alpha=0.15, label='Filtered Data')
            if self.analysis.specifiedPowerCurve is not None:
                has_spec_pc = len(self.analysis.specifiedPowerCurve.powerCurveLevels.index) != 0
            else:
                has_spec_pc = False
            if has_spec_pc:
                ax = self.analysis.specifiedPowerCurve.powerCurveLevels.sort_index()['Specified Power'].plot(ax = ax, color='#FF0000',alpha=0.9,label='Specified')
            if self.analysis.specifiedPowerCurve != self.analysis.powerCurve:
                if ((self.analysis.powerCurve.name != 'All Measured') and show_analysis_pc):
                    ax = self.analysis.powerCurve.powerCurveLevels.sort_index()['Actual Power'].plot(ax = ax, color='#A37ACC',alpha=0.9,label=self.analysis.powerCurve.name)
            meanPowerCurve = meanPowerCurveObj.powerCurveLevels[[windSpeedCol,powerCol,'Data Count']][self.analysis.allMeasuredPowerCurve.powerCurveLevels.loc[meanPowerCurveObj.powerCurveLevels.index, 'Data Count'] > 0].reset_index().set_index(windSpeedCol)
            ax = meanPowerCurve[powerCol].plot(ax = ax,color=mean_pc_color,alpha=0.95,linestyle='--',
                                  label=mean_title)
            ax.legend(loc=4, scatterpoints = 1)
            if has_spec_pc:
                ax.set_xlim([self.analysis.specifiedPowerCurve.powerCurveLevels.index.min(), self.analysis.specifiedPowerCurve.powerCurveLevels.index.max()+2.0])
            else:
                ax.set_xlim([min(df[windSpeedCol].min(),meanPowerCurve.index.min()), max(df[windSpeedCol].max(),meanPowerCurve.index.max()+2.0)])
            ax.set_xlabel(self.analysis.inputHubWindSpeedSource + ' (m/s)')
            ax.set_ylabel(powerCol + ' (kW)')
            if anon:
                ax.xaxis.set_ticklabels([])
                ax.yaxis.set_ticklabels([])
            fname = ("PowerCurve - " + powerCol + " vs " + windSpeedCol + ".png") if fname is None else fname
            file_out = self.path + os.sep + fname
            chckMake(self.path)
            plt.savefig(file_out)
            plt.close()
            return file_out
        except:
            raise
            print "Tried to make a power curve scatter chart for %s. Couldn't." % meanPowerCurveObj.name

    def plotTurbCorrectedPowerCurve(self, windSpeedCol, powerCol, meanPowerCurveObj):
        try:
            from matplotlib import pyplot as plt
            plt.ioff()
            if (windSpeedCol == self.analysis.densityCorrectedHubWindSpeed) or ((windSpeedCol == self.analysis.inputHubWindSpeed) and (self.analysis.densityCorrectionActive)):
                plotTitle = "Power Curve (corrected to {dens} kg/m^3)".format(dens=self.analysis.referenceDensity)
            else:
                plotTitle = "Power Curve"
            ax = self.analysis.dataFrame.plot(kind='scatter', x=windSpeedCol, y=powerCol, title=plotTitle, alpha=0.15, label='Filtered Data')
            if self.analysis.specifiedPowerCurve is not None:
                has_spec_pc = len(self.analysis.specifiedPowerCurve.powerCurveLevels.index) != 0
            else:
                has_spec_pc = False
            if has_spec_pc:
                ax = self.analysis.specifiedPowerCurve.powerCurveLevels.sort_index()['Specified Power'].plot(ax = ax, color='#FF0000',alpha=0.9,label='Specified')
            meanPowerCurve = meanPowerCurveObj.powerCurveLevels[[windSpeedCol,powerCol,'Data Count']][self.analysis.allMeasuredPowerCurve.powerCurveLevels['Data Count'] > 0 ].reset_index().set_index(windSpeedCol)
            ax = meanPowerCurve[powerCol].plot(ax = ax,color='#00FF00',alpha=0.95,linestyle='--',
                                  label='Mean Power Curve')
            ax2 = ax.twinx()
            if has_spec_pc:
                ax.set_xlim([self.analysis.specifiedPowerCurve.powerCurveLevels.index.min(), self.analysis.specifiedPowerCurve.powerCurveLevels.index.max()+2.0])
                ax2.set_xlim([self.analysis.specifiedPowerCurve.powerCurveLevels.index.min(), self.analysis.specifiedPowerCurve.powerCurveLevels.index.max()+2.0])
            else:
                ax.set_xlim([min(self.analysis.dataFrame[windSpeedCol].min(),meanPowerCurve.index.min()), max(self.analysis.dataFrame[windSpeedCol].max(),meanPowerCurve.index.max()+2.0)])
                ax2.set_xlim([min(self.analysis.dataFrame[windSpeedCol].min(),meanPowerCurve.index.min()), max(self.analysis.dataFrame[windSpeedCol].max(),meanPowerCurve.index.max()+2.0)])

            ax.set_xlabel(self.analysis.inputHubWindSpeedSource + ' (m/s)')
            ax.set_ylabel(powerCol + ' (kW)')
            refTurbCol = 'Specified Turbulence' if self.analysis.powerCurveMode == 'Specified' else self.analysis.hubTurbulence
            ax2.plot(self.analysis.powerCurve.powerCurveLevels.sort_index().index, self.analysis.powerCurve.powerCurveLevels.sort_index()[refTurbCol] * 100., 'm--', label = 'Reference TI')
            ax2.set_ylabel('Reference TI (%)')
            h1, l1 = ax.get_legend_handles_labels()
            h2, l2 = ax2.get_legend_handles_labels()
            ax.legend(h1+h2, l1+l2, loc=4, scatterpoints = 1)
            file_out = self.path + "/PowerCurve TI Corrected - " + powerCol + " vs " + windSpeedCol + ".png"
            chckMake(self.path)
            plt.savefig(file_out)
            plt.close()
            return file_out
        except:
            print "Tried to make a TI corrected power curve scatter chart for %s. Couldn't." % meanPowerCurveObj.name

    def plotPowerLimits(self):
        try:
            from matplotlib import pyplot as plt
            plt.ioff()
            windSpeedCol = self.analysis.densityCorrectedHubWindSpeed
            ax = self.analysis.dataFrame.plot(kind='scatter',x=windSpeedCol,y=self.analysis.actualPower ,title="Power Values Corrected to {dens} kg/m^3".format(dens=self.analysis.referenceDensity),alpha=0.5,label='Power Mean')
            ax = self.analysis.dataFrame.plot(ax=ax,kind='scatter',x=windSpeedCol,y="Power Min",alpha=0.2,label='Power Min',color = 'orange')
            ax = self.analysis.dataFrame.plot(ax=ax,kind='scatter',x=windSpeedCol,y="Power Max",alpha=0.2,label='Power Max',color = 'green')
            ax = self.analysis.dataFrame.plot(ax=ax,kind='scatter',x=windSpeedCol,y="Power SD",alpha=0.2,label='Power SD',color = 'purple')
            ax = self.analysis.specifiedPowerCurve.powerCurveLevels.sort_index()['Specified Power'].plot(ax = ax, color='#FF0000',alpha=0.9,label='Specified')
            ax.set_xlim([self.analysis.specifiedPowerCurve.powerCurveLevels.index.min(), self.analysis.specifiedPowerCurve.powerCurveLevels.index.max()+2.0])
            ax.legend(loc=4, scatterpoints = 1)
            ax.set_xlabel(windSpeedCol)
            ax.set_ylabel("Power [kW]")
            file_out = self.path + "/PowerValues.png"
            chckMake(self.path)
            plt.savefig(file_out)
            plt.close()
            return file_out
        except:
            print "Tried to make a full power scatter chart. Couldn't."

    def plotCalibrationSectors(self):
        for datasetConf in self.analysis.datasetConfigs:
            try:
                from matplotlib import pyplot as plt
                plt.ioff()
                df = datasetConf.data.calibrationCalculator.calibrationSectorDataframe[['pctSpeedUp','LowerLimit','UpperLimit']].rename(columns={'pctSpeedUp':'% Speed Up','LowerLimit':"IEC Lower",'UpperLimit':"IEC Upper"})
                df.plot(kind = 'line', title = 'Variation of wind speed ratio with direction', figsize = (12,8))
                plt.ylabel('Wind Speed Ratio (Vturb/Vref) as %')
                file_out = self.path + os.sep + 'Wind Speed Ratio with Direction - All Sectors {nm}.png'.format(nm=datasetConf.name)
                plt.savefig(file_out)
                df = df.loc[np.logical_and(df.index > datasetConf.data.fullDataFrame[datasetConf.data.referenceDirectionBin].min()-5.0 , df.index < datasetConf.data.fullDataFrame[datasetConf.data.referenceDirectionBin].max()+5.0),:]
                df.plot(kind = 'line', title = 'Variation of wind speed ratio with direction', figsize = (12,8))
                plt.ylabel('Wind Speed Ratio (Vturb/Vref) as %')
                file_out = self.path + os.sep + 'Wind Speed Ratio with Direction - Selected Sectors {nm}.png'.format(nm=datasetConf.name)
                chckMake(self.path)
                plt.savefig(file_out)
                plt.close('all')
            except:
                print "Tried to plot variation of wind speed ratio with direction. Couldn't."


"""
.. Copyright (c) 2014, 2015 Marshall Farrier
   license http://opensource.org/licenses/MIT

Common - generic functions (:mod:`pynance.common`)
==================================================

.. currentmodule:: pynance.common
"""

import pandas as pd

def featurize(equity_data, n_sessions, **kwargs):
    """
    Generate a raw (unnormalized) feature set from the input data.
    The value at `column` on the given date is taken
    as a feature, and each row contains values for n_sessions

    Parameters
    -----------
    equity_data : DataFrame
        data from which to generate features

    n_sessions : int
        number of sessions to use as features

    selection : str, default: 'Adj Close'
        column of `equity_data` from which to generate features.

    columns : list, default: ``map(str, range((-n_sessions + 1), 1))``
        column names for output DataFrame. Default will look like:
        ['-5', '-4', '-3', '-2', '-1', '0'].

    Returns
    ----------
    out : DataFrame
        Each row is a sequence of `n_sessions` session values where
        the last column matches the value on the date specified by
        the DataFrame index.

    Examples
    --------
    >>> pn.featurize(equity_data, n_sessions, **kwargs)
    """
    #Benchmarking
    #>>> s = 'from __main__ import data\nimport datetime as dt\n'
    #>>> timeit.timeit('data.featurize(data.get("ge", dt.date(1960, 1, 1),
    #        dt.date(2014, 12, 31)), 256)', setup=s, number=1)
    #1.6771750450134277
    columns = kwargs.get('columns', map(str, range(-n_sessions + 1, 1)))
    selection = kwargs.get('selection', 'Adj Close')
    # empty DataFrame with desired index and column labels
    features = pd.DataFrame(index=equity_data.index[(n_sessions - 1):],
            columns=columns, dtype='float64')
    values = equity_data[selection].values
    for i in range(n_sessions - 1):
        features.iloc[:, i] = values[i:(-n_sessions + i + 1)]
    features.iloc[:, n_sessions - 1] = values[(n_sessions - 1):]
    return features

def decorate(fn, *args, **kwargs):
    """
    Return a new function that replicates the behavior of the input
    but also returns an additional value. Used for creating functions
    of the proper type to pass to `labeledfeatures()`.

    Parameters
    ----------
    fn : function

    *args : any
        Additional parameters that the returned function will return

    **kwargs : dict
        Each element in `kwargs` will become an attribute of the output
        function.

    Returns
    ----------
    wrapped : function
        New function that acts like `fn` except that it also returns
        an additional value.

    Examples
    ----------
    >>> from functools import partial
    >>> forecast_interval = 32
    >>> features, labels = pn.data.labeledfeatures(eqdata, 256, featurefn,
    ...        decorate(partial(pn.data.lab.growth, forecast_interval, 'Adj Close'), forecast_interval))
    >>> def f():
    ...    return 0, 1
    ...
    >>> pn.decorate(f, 3, 4, 5)()
    (0, 1, 3, 4, 5)
    >>> pn.decorate(lambda x: x * .5, 3, 4, 5)(1.)
    (1., 3, 4, 5)
    >>> pn.decorate(lambda x: x, 1 2)('foo')
    ('foo', 1, 2)
    >>> pn.decorate(f, 'foo'):
    (0, 1, 'foo')
    pn.decorate(f, 0, foo='bar').foo
    >>> 'bar'

    Notes
    ----------
    If `fn` returns multiple values, these will be returned in sequence
    as the first values returned by `add_rets(fn, arg0, arg1, arg2)`. See example
    above.
    """
    def _wrapper(*_args, **kwargs):
        _ret = fn(*_args, **kwargs)
        if isinstance(_ret, tuple):
            return _ret + args
        if len(args) == 0:
            return _ret
        return (_ret,) + args
    for key, value in kwargs.items():
        _wrapper.__dict__[key] = value
    return _wrapper

def expand(fn, col, inputtype=pd.DataFrame):
    """
    Wrap a function applying to a single column to make a function
    applying to a multi-dimensional dataframe or ndarray

    Parameters
    ----------
    fn : function
        Function that applies to a series or vector.

    col : str or int
        Index of column to which to apply `fn`.

    inputtype : class or type
        Type of input to be expected by the wrapped function.
        Normally pd.DataFrame or np.ndarray. Defaults to pd.DataFrame.

    Returns
    ----------
    wrapped : function
        Function that takes an input of type `inputtype` and applies
        `fn` to the specified `col`.
    """
    if inputtype == pd.DataFrame:
        if isinstance(col, int):
            def _wrapper(*args, **kwargs):
                return fn(args[0].iloc[:, col], *args[1:], **kwargs)
            return _wrapper
        def _wrapper(*args, **kwargs):
            return fn(args[0].loc[:, col], *args[1:], **kwargs)
        return _wrapper
    elif inputtype == np.ndarray:
        def _wrapper(*args, **kwargs):
            return fn(args[0][:, col], *args[1:], **kwargs)
        return _wrapper
    raise TypeError("invalid input type")

def has_na(eqdata):
    """
    Return false if `eqdata` contains no missing values.

    Parameters
    ----------
    eqdata : DataFrame or ndarray
        Data to check for missing values (NaN, None)

    Returns
    ----------
    answer : bool
        False iff `eqdata` contains no missing values.
    """
    if isinstance(eqdata, pd.DataFrame):
        _values = eqdata.values
    else:
        _values = eqdata
    return len(_values[pd.isnull(_values)]) > 0


# Author: Christian Osendorfer <osendorf@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD3

import numpy as np

from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.exceptions import ConvergenceWarning
from sklearn.decomposition import FactorAnalysis
from sklearn.utils.testing import ignore_warnings

# Ignore warnings from switching to more power iterations in randomized_svd
@ignore_warnings
def test_factor_analysis():
    # Test FactorAnalysis ability to recover the data covariance structure
    rng = np.random.RandomState(0)
    n_samples, n_features, n_components = 20, 5, 3

    # Some random settings for the generative model
    W = rng.randn(n_components, n_features)
    # latent variable of dim 3, 20 of it
    h = rng.randn(n_samples, n_components)
    # using gamma to model different noise variance
    # per component
    noise = rng.gamma(1, size=n_features) * rng.randn(n_samples, n_features)

    # generate observations
    # wlog, mean is 0
    X = np.dot(h, W) + noise

    assert_raises(ValueError, FactorAnalysis, svd_method='foo')
    fa_fail = FactorAnalysis()
    fa_fail.svd_method = 'foo'
    assert_raises(ValueError, fa_fail.fit, X)
    fas = []
    for method in ['randomized', 'lapack']:
        fa = FactorAnalysis(n_components=n_components, svd_method=method)
        fa.fit(X)
        fas.append(fa)

        X_t = fa.transform(X)
        assert_equal(X_t.shape, (n_samples, n_components))

        assert_almost_equal(fa.loglike_[-1], fa.score_samples(X).sum())
        assert_almost_equal(fa.score_samples(X).mean(), fa.score(X))

        diff = np.all(np.diff(fa.loglike_))
        assert_greater(diff, 0., 'Log likelihood dif not increase')

        # Sample Covariance
        scov = np.cov(X, rowvar=0., bias=1.)

        # Model Covariance
        mcov = fa.get_covariance()
        diff = np.sum(np.abs(scov - mcov)) / W.size
        assert_less(diff, 0.1, "Mean absolute difference is %f" % diff)
        fa = FactorAnalysis(n_components=n_components,
                            noise_variance_init=np.ones(n_features))
        assert_raises(ValueError, fa.fit, X[:, :2])

    f = lambda x, y: np.abs(getattr(x, y))  # sign will not be equal
    fa1, fa2 = fas
    for attr in ['loglike_', 'components_', 'noise_variance_']:
        assert_almost_equal(f(fa1, attr), f(fa2, attr))

    fa1.max_iter = 1
    fa1.verbose = True
    assert_warns(ConvergenceWarning, fa1.fit, X)

    # Test get_covariance and get_precision with n_components == n_features
    # with n_components < n_features and with n_components == 0
    for n_components in [0, 2, X.shape[1]]:
        fa.n_components = n_components
        fa.fit(X)
        cov = fa.get_covariance()
        precision = fa.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]), 12)


import copy
import echidna
import echidna.output.plot as plot
import echidna.core.spectra as spectra
from echidna.output import store
import matplotlib.pyplot as plt

import argparse
import glob
import numpy as np
import os

def convertor(path):
    flist=np.array(glob.glob(path))
    for ntuple in flist:
        os.system("python ~/echidna/echidna/scripts/dump_spectra_ntuple.py -c ~/workspace/PhD/fitting/config.yml -f "+ str(ntuple)+" -s hdf5/")

def combinerNtuple(path,filename):
    flist=np.array(glob.glob(path))
    print flist
    first = True
    for hdf5 in flist:
        print hdf5
        if first:
            spectrum1 = store.fill_from_ntuple(hdf5)
            first = False
        else:
            spectrum2 = store.fill_from_ntuple(hdf5)
            spectrum1.add(spectrum2)
    store.dump(filename, spectrum1)

def combiner(path,filename):
    flist=np.array(glob.glob(path))
    print flist
    first = True
    for hdf5 in flist:
        print hdf5
        if first:
            spectrum1 = store.load(hdf5)
            first = False
        else:
            spectrum2 = store.load(hdf5)
            spectrum1.add(spectrum2)
    store.dump(filename, spectrum1)

"""The way you should do it is to define a lot of spectra and then plot them.
    You don't really know how to normlise the histrogram or indeed weather that is of any uses in the first
    place.
"""

def slicer(spectrumPath,filler,nslice):
    for i in range(nslice):
       spectrum=store.load(spectrumPath)
       print spectrum.sum()
       shrink_dict = {"energy_reco_low": 0.,
                      "energy_reco_high": 0.6,
                      "radial_reco_low": i*6000.0/nslice,
                      "radial_reco_high": (i+1)*6000/nslice}
       spectrum.cut(**shrink_dict)
       spectrum.scale(1)
       spec2=copy.copy(spectrum)
       spec2._name=str(i*1000)+"mm to "+str((i+1)*1000)+"mm"
       print type(spec2)
       filler.append(spec2)

def slicerMC(spectrumPath,filler,nslice):
    for i in range(nslice):
       spectrum=store.load(spectrumPath)
       print spectrum.sum()
       shrink_dict = {"energy_mc_low": 0.,
                      "energy_mc_high": 1,
                      "radial_mc_low": i*6000.0/nslice,
                      "radial_mc_high": (i+1)*6000/nslice}
       spectrum.cut(**shrink_dict)
       spectrum.scale(1)
       spec2=copy.copy(spectrum)
       spec2._name="MC"
       print type(spec2)
       print "This gives the number os events in each window:"
       print "mc : "+str(i*6000.0/nslice)+"mm to "+str((i+1)*6000.0/nslice)+"mm : "+str(spec2.sum())
       filler.append(spec2)

def slicerReco(spectrumPath,filler,nslice):
    for i in range(nslice):
       spectrum=store.load(spectrumPath)
       print spectrum.sum()
       shrink_dict = {"energy_reco_low": 0.,
                      "energy_reco_high": 1.,
                      "radial_reco_low": i*6000.0/nslice,
                      "radial_reco_high": (i+1)*6000/nslice}
       spectrum.cut(**shrink_dict)
       spectrum.scale(1)
       spec2=copy.copy(spectrum)
       spec2._name="Reco"
       print type(spec2)
       print "This gives the number os events in each window:"
       print "reco : "+str(i*6000.0/nslice)+"mm to "+str((i+1)*6000.0/nslice)+"mm : "+str(spec2.sum())
       filler.append(spec2)

def signalPlotter(spectra,dim,name):
    i=0
    for spec in spectra:
        fig = plt.figure()
        ax= fig.add_subplot(1,1,1)

        par = spec.get_config().get_par(dim)
        width = par.get_width()
        bins = np.linspace(par._low,par._high, par._bins+1)
        x = bins[:-1] + 0.5*width

        plt.xlabel(str(dim)+ " [" + par.get_unit() + "]")
        plt.ylabel("Events per " + str(width) + " " + par.get_unit() + " bin")

        ax.set(title="Normalised energy spectrum in "+str(i*1000)+"mm to "+str((i+1)*1000)+"mm ",ylabel="Events per " + str(width) + " " + par.get_unit() + " bin", xlabel=str(dim)+" [" + par.get_unit() + "]")
        ax.hist(x,bins,weights=spec.project(dim),histtype="stepfilled", color="RoyalBlue",label=spec._name)
        fig.savefig("slice_"+str(name)+"_"+str(i*1000)+"_"+str((i+1)*1000)+".png")
        i=1+i

def combiPlotter(spectra,dim,name):
    i=0
    fig = plt.figure()
    ax= fig.add_subplot(1,1,1)
    for spec in spectra:
        par = spec.get_config().get_par(dim)
        width = par.get_width()
        bins = np.linspace(par._low,par._high, par._bins+1)
        x = bins[:-1] + 0.5*width

        plt.xlabel(str(dim)+ " [" + par.get_unit() + "]")
        plt.ylabel("Events per " + str(width) + " " + par.get_unit() + " bin")

        ax.set(title="Normalised energy spectrum in 1000mm slices",ylabel="Events per " + str(width) + " " + par.get_unit() + " bin", xlabel="energy_reco"+ " [" + par.get_unit() + "]")
        ax.hist(x,bins,weights=spec.project("energy_reco"),label=spec._name,histtype='step')
        ax.set_ylim([0,0.03])
        ax.set_xlim([0.2,0.7])
        ax.legend(loc="best")
    fig.savefig("combined_"+str(name)+".png")

def func(path,nslice,name):
    spectra=[]
    slicer(path,spectra,nslice)
    signalPlotter(spectra,"energy_reco",name)
    combiPlotter(spectra,"energy_reco",name)

def po210():

    convertor("po210_ntuple/*")
    combiner("hdf5/SolarPo**ntuple*","hdf5/SolarPo210_combined.hdf5")

    plotpath="plots/"

    func("hdf5/SolarPo210_combined.hdf5",6,"po210")

def bi210():
    convertor("bi210_ntuple/*")
    combiner("hdf5/SolarBi**ntuple*","hdf5/SolarBi210_combined.hdf5")
    plotpath="plots/"
    func("hdf5/SolarBi210_combined.hdf5",6,"bi210")

def compair(spectrumPathReco,spectrumPathMC,name):
    spectraReco=[]
    spectraMC=[]
    slicerReco(spectrumPathReco,spectraReco,6)
    slicerMC(spectrumPathMC,spectraMC,6)

    for i in range(0,len(spectraReco)):
        fig = plt.figure()
        ax= fig.add_subplot(1,1,1)

        par = spectraReco[i].get_config().get_par("energy_reco")
        width = par.get_width()
        bins = np.linspace(par._low,par._high, par._bins+1)
        x = bins[:-1] + 0.5*width

        ax.set(title="Normalised energy spectrum in "+str(i*1000)+"mm to "+str((i+1)*1000)+"mm ",ylabel="Events per " + str(width) + " " + par.get_unit() + " bin", xlabel="Energy [" + par.get_unit() + "]")
        ax.hist(x,bins,weights=spectraReco[i].project("energy_reco"),histtype="stepfilled",label=spectraReco[i]._name)

        par = spectraMC[i].get_config().get_par("energy_mc")
        width = par.get_width()
        bins = np.linspace(par._low,par._high, par._bins+1)
        x = bins[:-1] + 0.5*width

        ax.hist(x,bins,weights=spectraMC[i].project("energy_mc"),histtype="stepfilled",label=spectraMC[i]._name,alpha=0.75)
        ax.legend(loc=2)
        fig.savefig("compare_"+str(name)+"_"+str(i*1000)+"_"+str((i+1)*1000)+".png")

if __name__=="__main__":

    print "You need to compare the recon against the mc"
    print "You should bin in bigger bins becuase you could then bin in 4d"
    """You need to plot the standard spectra"""


#!/usr/bin/env python
# vim:ts=4:sts=4:sw=4:et:wrap:ai:fileencoding=utf-8:

import collections
#import matplotlib.pyplot as plt

factor = 1/4

class TraceGenerator():

    def __init__(self):
        fname='/Users/jobelenus/work/thegreco/cpu.entries'
        self.fname = fname
        with open(self.fname) as f:
            self.lines = f.readlines()
            self.cpu = map(int, self.lines)

    def gen_cpu_trace(self):
        return self.cpu

    def gen_mem_trace(self):
        self.mem = collections.deque(self.cpu)
        self.mem.rotate(len(self.cpu)/4)
        return self.mem

    def gen_disk_trace(self):
        self.disk = collections.deque(self.cpu)
        self.disk.rotate(2*len(self.cpu)/4)
        return self.disk

    def gen_net_trace(self):
        self.net = collections.deque(self.cpu)
        self.net.rotate(3*len(self.cpu)/4)
        return self.net

    def gen_trace(self):
        self.gen_cpu_trace()
        self.gen_mem_trace()
        self.gen_disk_trace()
        self.gen_net_trace()
        self.trace = zip(self.cpu, self.mem, self.disk, self.net)
        return self.trace

#tg = TraceGenerator()
#cpu = tg.gen_cpu_trace()
#mem = tg.gen_mem_trace()
#disk = tg.gen_disk_trace()
#net = tg.gen_net_trace()
#trace = zip(cpu, mem, disk, net)

#print trace
#plt.bar(range(0,len(cpu)), cpu)
#plt.show()


import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises

from sklearn.decomposition import PCA, KernelPCA
from sklearn.datasets import make_circles
from sklearn.linear_model import Perceptron
from sklearn.utils.testing import assert_less
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.metrics.pairwise import rbf_kernel

def test_kernel_pca():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    for eigen_solver in ("auto", "dense", "arpack"):
        for kernel in ("linear", "rbf", "poly"):
            # transform fit data
            kpca = KernelPCA(4, kernel=kernel, eigen_solver=eigen_solver,
                             fit_inverse_transform=True)
            X_fit_transformed = kpca.fit_transform(X_fit)
            X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit)
            assert_array_almost_equal(np.abs(X_fit_transformed),
                                      np.abs(X_fit_transformed2))

            # transform new data
            X_pred_transformed = kpca.transform(X_pred)
            assert_equal(X_pred_transformed.shape[1],
                         X_fit_transformed.shape[1])

            # inverse transform
            X_pred2 = kpca.inverse_transform(X_pred_transformed)
            assert_equal(X_pred2.shape, X_pred.shape)

def test_invalid_parameters():
    assert_raises(ValueError, KernelPCA, 10, fit_inverse_transform=True,
                  kernel='precomputed')

def test_kernel_pca_sparse():
    rng = np.random.RandomState(0)
    X_fit = sp.csr_matrix(rng.random_sample((5, 4)))
    X_pred = sp.csr_matrix(rng.random_sample((2, 4)))

    for eigen_solver in ("auto", "arpack"):
        for kernel in ("linear", "rbf", "poly"):
            # transform fit data
            kpca = KernelPCA(4, kernel=kernel, eigen_solver=eigen_solver,
                             fit_inverse_transform=False)
            X_fit_transformed = kpca.fit_transform(X_fit)
            X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit)
            assert_array_almost_equal(np.abs(X_fit_transformed),
                                      np.abs(X_fit_transformed2))

            # transform new data
            X_pred_transformed = kpca.transform(X_pred)
            assert_equal(X_pred_transformed.shape[1],
                    X_fit_transformed.shape[1])

            # inverse transform
            #X_pred2 = kpca.inverse_transform(X_pred_transformed)
            #assert_equal(X_pred2.shape, X_pred.shape)

def test_kernel_pca_linear_kernel():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    # for a linear kernel, kernel PCA should find the same projection as PCA
    # modulo the sign (direction)
    # fit only the first four components: fifth is near zero eigenvalue, so
    # can be trimmed due to roundoff error
    assert_array_almost_equal(
        np.abs(KernelPCA(4).fit(X_fit).transform(X_pred)),
        np.abs(PCA(4).fit(X_fit).transform(X_pred)))

def test_kernel_pca_n_components():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    for eigen_solver in ("dense", "arpack"):
        for c in [1, 2, 4]:
            kpca = KernelPCA(n_components=c, eigen_solver=eigen_solver)
            shape = kpca.fit(X_fit).transform(X_pred).shape

            assert_equal(shape, (2, c))

def test_kernel_pca_precomputed():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((5, 4))
    X_pred = rng.random_sample((2, 4))

    for eigen_solver in ("dense", "arpack"):
        X_kpca = KernelPCA(4, eigen_solver=eigen_solver).\
            fit(X_fit).transform(X_pred)
        X_kpca2 = KernelPCA(4, eigen_solver=eigen_solver,
                kernel='precomputed').fit(np.dot(X_fit,
                    X_fit.T)).transform(np.dot(X_pred, X_fit.T))

        X_kpca_train = KernelPCA(4, eigen_solver=eigen_solver,
                kernel='precomputed').fit_transform(np.dot(X_fit, X_fit.T))
        X_kpca_train2 = KernelPCA(4, eigen_solver=eigen_solver,
                kernel='precomputed').fit(np.dot(X_fit,
                    X_fit.T)).transform(np.dot(X_fit, X_fit.T))

        assert_array_almost_equal(np.abs(X_kpca),
                                  np.abs(X_kpca2))

        assert_array_almost_equal(np.abs(X_kpca_train),
                                  np.abs(X_kpca_train2))

def test_kernel_pca_invalid_kernel():
    rng = np.random.RandomState(0)
    X_fit = rng.random_sample((2, 4))
    kpca = KernelPCA(kernel="tototiti")
    assert_raises(ValueError, kpca.fit, X_fit)

def test_gridsearch_pipeline():
    # Test if we can do a grid-search to find parameters to separate
    # circles with a perceptron model.
    X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                        random_state=0)
    kpca = KernelPCA(kernel="rbf", n_components=2)
    pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron())])
    param_grid = dict(kernel_pca__gamma=2. ** np.arange(-2, 2))
    grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid)
    grid_search.fit(X, y)
    assert_equal(grid_search.best_score_, 1)

def test_gridsearch_pipeline_precomputed():
    # Test if we can do a grid-search to find parameters to separate
    # circles with a perceptron model using a precomputed kernel.
    X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                        random_state=0)
    kpca = KernelPCA(kernel="precomputed", n_components=2)
    pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron())])
    param_grid = dict(Perceptron__n_iter=np.arange(1, 5))
    grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid)
    X_kernel = rbf_kernel(X, gamma=2.)
    grid_search.fit(X_kernel, y)
    assert_equal(grid_search.best_score_, 1)

def test_nested_circles():
    """Test the linear separability of the first 2D KPCA transform"""
    X, y = make_circles(n_samples=400, factor=.3, noise=.05,
                        random_state=0)

    # 2D nested circles are not linearly separable
    train_score = Perceptron().fit(X, y).score(X, y)
    assert_less(train_score, 0.8)

    # Project the circles data into the first 2 components of a RBF Kernel
    # PCA model.
    # Note that the gamma value is data dependent. If this test breaks
    # and the gamma value has to be updated, the Kernel PCA example will
    # have to be updated too.
    kpca = KernelPCA(kernel="rbf", n_components=2,
                     fit_inverse_transform=True, gamma=2.)
    X_kpca = kpca.fit_transform(X)

    # The data is perfectly linearly separable in that space
    train_score = Perceptron().fit(X_kpca, y).score(X_kpca, y)
    assert_equal(train_score, 1.0)

if __name__ == '__main__':
    import nose
    nose.run(argv=['', __file__])


# Author: Vlad Niculae
# Licence: BSD 3 clause

import numpy as np

from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import ignore_warnings

from sklearn.linear_model import (orthogonal_mp, orthogonal_mp_gram,
                                  OrthogonalMatchingPursuit,
                                  OrthogonalMatchingPursuitCV,
                                  LinearRegression)
from sklearn.utils import check_random_state
from sklearn.datasets import make_sparse_coded_signal

n_samples, n_features, n_nonzero_coefs, n_targets = 20, 30, 5, 3
y, X, gamma = make_sparse_coded_signal(n_targets, n_features, n_samples,
                                       n_nonzero_coefs, random_state=0)
G, Xy = np.dot(X.T, X), np.dot(X.T, y)
# this makes X (n_samples, n_features)
# and y (n_samples, 3)

def test_correct_shapes():
    assert_equal(orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5).shape,
                 (n_features,))
    assert_equal(orthogonal_mp(X, y, n_nonzero_coefs=5).shape,
                 (n_features, 3))

def test_correct_shapes_gram():
    assert_equal(orthogonal_mp_gram(G, Xy[:, 0], n_nonzero_coefs=5).shape,
                 (n_features,))
    assert_equal(orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5).shape,
                 (n_features, 3))

def test_n_nonzero_coefs():
    assert_true(np.count_nonzero(orthogonal_mp(X, y[:, 0],
                                 n_nonzero_coefs=5)) <= 5)
    assert_true(np.count_nonzero(orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5,
                                               precompute=True)) <= 5)

def test_tol():
    tol = 0.5
    gamma = orthogonal_mp(X, y[:, 0], tol=tol)
    gamma_gram = orthogonal_mp(X, y[:, 0], tol=tol, precompute=True)
    assert_true(np.sum((y[:, 0] - np.dot(X, gamma)) ** 2) <= tol)
    assert_true(np.sum((y[:, 0] - np.dot(X, gamma_gram)) ** 2) <= tol)

def test_with_without_gram():
    assert_array_almost_equal(
        orthogonal_mp(X, y, n_nonzero_coefs=5),
        orthogonal_mp(X, y, n_nonzero_coefs=5, precompute=True))

def test_with_without_gram_tol():
    assert_array_almost_equal(
        orthogonal_mp(X, y, tol=1.),
        orthogonal_mp(X, y, tol=1., precompute=True))

def test_unreachable_accuracy():
    assert_array_almost_equal(
        orthogonal_mp(X, y, tol=0),
        orthogonal_mp(X, y, n_nonzero_coefs=n_features))

    assert_array_almost_equal(
        assert_warns(RuntimeWarning, orthogonal_mp, X, y, tol=0,
                     precompute=True),
        orthogonal_mp(X, y, precompute=True,
                      n_nonzero_coefs=n_features))

def test_bad_input():
    assert_raises(ValueError, orthogonal_mp, X, y, tol=-1)
    assert_raises(ValueError, orthogonal_mp, X, y, n_nonzero_coefs=-1)
    assert_raises(ValueError, orthogonal_mp, X, y,
                  n_nonzero_coefs=n_features + 1)
    assert_raises(ValueError, orthogonal_mp_gram, G, Xy, tol=-1)
    assert_raises(ValueError, orthogonal_mp_gram, G, Xy, n_nonzero_coefs=-1)
    assert_raises(ValueError, orthogonal_mp_gram, G, Xy,
                  n_nonzero_coefs=n_features + 1)

def test_perfect_signal_recovery():
    idx, = gamma[:, 0].nonzero()
    gamma_rec = orthogonal_mp(X, y[:, 0], 5)
    gamma_gram = orthogonal_mp_gram(G, Xy[:, 0], 5)
    assert_array_equal(idx, np.flatnonzero(gamma_rec))
    assert_array_equal(idx, np.flatnonzero(gamma_gram))
    assert_array_almost_equal(gamma[:, 0], gamma_rec, decimal=2)
    assert_array_almost_equal(gamma[:, 0], gamma_gram, decimal=2)

def test_estimator():
    omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_nonzero_coefs)
    omp.fit(X, y[:, 0])
    assert_equal(omp.coef_.shape, (n_features,))
    assert_equal(omp.intercept_.shape, ())
    assert_true(np.count_nonzero(omp.coef_) <= n_nonzero_coefs)

    omp.fit(X, y)
    assert_equal(omp.coef_.shape, (n_targets, n_features))
    assert_equal(omp.intercept_.shape, (n_targets,))
    assert_true(np.count_nonzero(omp.coef_) <= n_targets * n_nonzero_coefs)

    omp.set_params(fit_intercept=False, normalize=False)

    omp.fit(X, y[:, 0])
    assert_equal(omp.coef_.shape, (n_features,))
    assert_equal(omp.intercept_, 0)
    assert_true(np.count_nonzero(omp.coef_) <= n_nonzero_coefs)

    omp.fit(X, y)
    assert_equal(omp.coef_.shape, (n_targets, n_features))
    assert_equal(omp.intercept_, 0)
    assert_true(np.count_nonzero(omp.coef_) <= n_targets * n_nonzero_coefs)

def test_identical_regressors():
    newX = X.copy()
    newX[:, 1] = newX[:, 0]
    gamma = np.zeros(n_features)
    gamma[0] = gamma[1] = 1.
    newy = np.dot(newX, gamma)
    assert_warns(RuntimeWarning, orthogonal_mp, newX, newy, 2)

def test_swapped_regressors():
    gamma = np.zeros(n_features)
    # X[:, 21] should be selected first, then X[:, 0] selected second,
    # which will take X[:, 21]'s place in case the algorithm does
    # column swapping for optimization (which is the case at the moment)
    gamma[21] = 1.0
    gamma[0] = 0.5
    new_y = np.dot(X, gamma)
    new_Xy = np.dot(X.T, new_y)
    gamma_hat = orthogonal_mp(X, new_y, 2)
    gamma_hat_gram = orthogonal_mp_gram(G, new_Xy, 2)
    assert_array_equal(np.flatnonzero(gamma_hat), [0, 21])
    assert_array_equal(np.flatnonzero(gamma_hat_gram), [0, 21])

def test_no_atoms():
    y_empty = np.zeros_like(y)
    Xy_empty = np.dot(X.T, y_empty)
    gamma_empty = ignore_warnings(orthogonal_mp)(X, y_empty, 1)
    gamma_empty_gram = ignore_warnings(orthogonal_mp)(G, Xy_empty, 1)
    assert_equal(np.all(gamma_empty == 0), True)
    assert_equal(np.all(gamma_empty_gram == 0), True)

def test_omp_path():
    path = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=True)
    last = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=False)
    assert_equal(path.shape, (n_features, n_targets, 5))
    assert_array_almost_equal(path[:, :, -1], last)
    path = orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5, return_path=True)
    last = orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5, return_path=False)
    assert_equal(path.shape, (n_features, n_targets, 5))
    assert_array_almost_equal(path[:, :, -1], last)

def test_omp_return_path_prop_with_gram():
    path = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=True,
                         precompute=True)
    last = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=False,
                         precompute=True)
    assert_equal(path.shape, (n_features, n_targets, 5))
    assert_array_almost_equal(path[:, :, -1], last)

def test_omp_cv():
    y_ = y[:, 0]
    gamma_ = gamma[:, 0]
    ompcv = OrthogonalMatchingPursuitCV(normalize=True, fit_intercept=False,
                                        max_iter=10, cv=5)
    ompcv.fit(X, y_)
    assert_equal(ompcv.n_nonzero_coefs_, n_nonzero_coefs)
    assert_array_almost_equal(ompcv.coef_, gamma_)
    omp = OrthogonalMatchingPursuit(normalize=True, fit_intercept=False,
                                    n_nonzero_coefs=ompcv.n_nonzero_coefs_)
    omp.fit(X, y_)
    assert_array_almost_equal(ompcv.coef_, omp.coef_)

def test_omp_reaches_least_squares():
    # Use small simple data; it's a sanity check but OMP can stop early
    rng = check_random_state(0)
    n_samples, n_features = (10, 8)
    n_targets = 3
    X = rng.randn(n_samples, n_features)
    Y = rng.randn(n_samples, n_targets)
    omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_features)
    lstsq = LinearRegression()
    omp.fit(X, Y)
    lstsq.fit(X, Y)
    assert_array_almost_equal(omp.coef_, lstsq.coef_)


# -*- coding: utf-8 -*-
import gc
import pandas
from datetime import datetime

from pandas import DataFrame

from sqlalchemy import create_engine
from sqlalchemy.sql import text
from sqlalchemy.orm import sessionmaker

from mining.utils import conf, log_it
from mining.utils._pandas import fix_render
from mining.db import DataWarehouse

from bottle.ext.mongo import MongoPlugin

class Cube(object):
    def __init__(self, _cube):

        log_it("START: {}".format(_cube['slug']), "bin-mining")

        self.mongo = MongoPlugin(
            uri=conf("mongodb")["uri"],
            db=conf("mongodb")["db"],
            json_mongo=True).get_mongo()

        try:
            del _cube['_id']
        except KeyError:
            pass
        self.cube = _cube
        self.slug = self.cube['slug']

    def load(self):
        self.cube['run'] = 'run'
        self.mongo['cube'].update({'slug': self.slug}, self.cube)

        self.cube['start_process'] = datetime.now()

        _sql = self.cube['sql']
        if _sql[-1] == ';':
            _sql = _sql[:-1]
        self.sql = u"""SELECT * FROM ({}) AS CUBE;""".format(_sql)

        self.connection = self.mongo['connection'].find_one({
            'slug': self.cube['connection']})['connection']

        log_it("CONNECT IN RELATION DATA BASE: {}".format(self.slug),
               "bin-mining")
        if 'sqlite' in self.connection:
            e = create_engine(self.connection)
        else:
            e = create_engine(self.connection,
                              **conf('openmining')['sql_conn_params'])
        Session = sessionmaker(bind=e)
        session = Session()

        resoverall = session.execute(text(self.sql))
        self.data = resoverall.fetchall()
        self.keys = resoverall.keys()

    def environment(self, t):
        if t not in ['relational']:
            self.sql = t

    def _data(self, data):
        self.data = data

    def _keys(self, keys):
        if type(keys) == list:
            self.keys = keys
        self.keys = list(keys)

    def frame(self, data_type=None):
        log_it("LOAD DATA ON DATAWAREHOUSE via {}: {}".format(
            data_type or 'dict', self.slug), "bin-mining")
        if data_type:
            self.df = getattr(pandas, "read_{}".format(data_type))(self.data)
        else:
            self.df = DataFrame(self.data)

        if self.df.empty:
            self.pdict = {}
            log_it('[warning]Empty cube: {}!!'.format(self.cube),
                   "bin-mining")
            return

        try:
            self.df.columns = self.keys
        except AttributeError:
            self._keys(self.df.columns.tolist())

        # If the OML is active, it renders the script that there is
        if conf("oml").get("on") and self.cube.get("oml"):
            from oml import RunTime
            self.df.columns = self.keys
            df = RunTime(conf("oml").get("language", "lua"),
                         self.df.to_dict(orient='records'),
                         self.cube.get("oml"),
                         conf("oml").get("class", {"OML": "oml.base.OMLBase"}))
            self.df = DataFrame(df)
            self._keys(self.df.columns.tolist())

        self.df.head()
        self.pdict = map(fix_render, self.df.to_dict(orient='records'))

    def save(self):
        log_it("SAVE DATA (JSON) ON DATA WAREHOUSE: {}".format(self.slug),
               "bin-mining")
        data = {'data': self.pdict, 'columns': self.keys}
        DW = DataWarehouse()
        DW.save(self.slug, data)

        self.cube['status'] = True
        self.cube['lastupdate'] = datetime.now()
        self.cube['run'] = True
        self.mongo['cube'].update({'slug': self.cube['slug']}, self.cube)

        log_it("CLEAN MEMORY: {}".format(self.slug), "bin-mining")
        gc.collect()


# -*- coding: utf-8 -*-
"""IPython Test Process Controller

This module runs one or more subprocesses which will actually run the IPython
test suite.

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2009-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from __future__ import print_function

import argparse
import json
import multiprocessing.pool
import os
import shutil
import signal
import sys
import subprocess
import time

from .iptest import have, test_group_names as py_test_group_names, test_sections, StreamCapturer
from IPython.utils.path import compress_user
from IPython.utils.py3compat import bytes_to_str
from IPython.utils.sysinfo import get_sys_info
from IPython.utils.tempdir import TemporaryDirectory

class TestController(object):
    """Run tests in a subprocess
    """
    #: str, IPython test suite to be executed.
    section = None
    #: list, command line arguments to be executed
    cmd = None
    #: dict, extra environment variables to set for the subprocess
    env = None
    #: list, TemporaryDirectory instances to clear up when the process finishes
    dirs = None
    #: subprocess.Popen instance
    process = None
    #: str, process stdout+stderr
    stdout = None

    def __init__(self):
        self.cmd = []
        self.env = {}
        self.dirs = []

    def setup(self):
        """Create temporary directories etc.

        This is only called when we know the test group will be run. Things
        created here may be cleaned up by self.cleanup().
        """
        pass

    def launch(self, buffer_output=False):
        # print('*** ENV:', self.env)  # dbg
        # print('*** CMD:', self.cmd)  # dbg
        env = os.environ.copy()
        env.update(self.env)
        output = subprocess.PIPE if buffer_output else None
        stdout = subprocess.STDOUT if buffer_output else None
        self.process = subprocess.Popen(self.cmd, stdout=output,
                stderr=stdout, env=env)

    def wait(self):
        self.stdout, _ = self.process.communicate()
        return self.process.returncode

    def print_extra_info(self):
        """Print extra information about this test run.

        If we're running in parallel and showing the concise view, this is only
        called if the test group fails. Otherwise, it's called before the test
        group is started.

        The base implementation does nothing, but it can be overridden by
        subclasses.
        """
        return

    def cleanup_process(self):
        """Cleanup on exit by killing any leftover processes."""
        subp = self.process
        if subp is None or (subp.poll() is not None):
            return  # Process doesn't exist, or is already dead.

        try:
            print('Cleaning up stale PID: %d' % subp.pid)
            subp.kill()
        except: # (OSError, WindowsError) ?
            # This is just a best effort, if we fail or the process was
            # really gone, ignore it.
            pass
        else:
            for i in range(10):
                if subp.poll() is None:
                    time.sleep(0.1)
                else:
                    break

        if subp.poll() is None:
            # The process did not die...
            print('... failed. Manual cleanup may be required.')

    def cleanup(self):
        "Kill process if it's still alive, and clean up temporary directories"
        self.cleanup_process()
        for td in self.dirs:
            td.cleanup()

    __del__ = cleanup

class PyTestController(TestController):
    """Run Python tests using IPython.testing.iptest"""
    #: str, Python command to execute in subprocess
    pycmd = None

    def __init__(self, section, options):
        """Create new test runner."""
        TestController.__init__(self)
        self.section = section
        # pycmd is put into cmd[2] in PyTestController.launch()
        self.cmd = [sys.executable, '-c', None, section]
        self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
        self.options = options

    def setup(self):
        ipydir = TemporaryDirectory()
        self.dirs.append(ipydir)
        self.env['IPYTHONDIR'] = ipydir.name
        self.workingdir = workingdir = TemporaryDirectory()
        self.dirs.append(workingdir)
        self.env['IPTEST_WORKING_DIR'] = workingdir.name
        # This means we won't get odd effects from our own matplotlib config
        self.env['MPLCONFIGDIR'] = workingdir.name

        # From options:
        if self.options.xunit:
            self.add_xunit()
        if self.options.coverage:
            self.add_coverage()
        self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams
        self.cmd.extend(self.options.extra_args)

    @property
    def will_run(self):
        try:
            return test_sections[self.section].will_run
        except KeyError:
            return True

    def add_xunit(self):
        xunit_file = os.path.abspath(self.section + '.xunit.xml')
        self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])

    def add_coverage(self):
        try:
            sources = test_sections[self.section].includes
        except KeyError:
            sources = ['IPython']

        coverage_rc = ("[run]\n"
                       "data_file = {data_file}\n"
                       "source =\n"
                       "  {source}\n"
                      ).format(data_file=os.path.abspath('.coverage.'+self.section),
                               source="\n  ".join(sources))
        config_file = os.path.join(self.workingdir.name, '.coveragerc')
        with open(config_file, 'w') as f:
            f.write(coverage_rc)

        self.env['COVERAGE_PROCESS_START'] = config_file
        self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd

    def launch(self, buffer_output=False):
        self.cmd[2] = self.pycmd
        super(PyTestController, self).launch(buffer_output=buffer_output)

js_prefix = 'js/'

def get_js_test_dir():
    import IPython.html.tests as t
    return os.path.join(os.path.dirname(t.__file__), '')

def all_js_groups():
    import glob
    test_dir = get_js_test_dir()
    all_subdirs = glob.glob(test_dir + '*/')
    return [js_prefix+os.path.relpath(x, test_dir) for x in all_subdirs if os.path.relpath(x, test_dir) != '__pycache__']

class JSController(TestController):
    """Run CasperJS tests """
    def __init__(self, section):
        """Create new test runner."""
        TestController.__init__(self)
        self.section = section
        js_test_dir = get_js_test_dir()
        includes = '--includes=' + os.path.join(js_test_dir,'util.js')
        test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
        self.cmd = ['casperjs', 'test', includes, test_cases]

    def setup(self):
        self.ipydir = TemporaryDirectory()
        self.nbdir = TemporaryDirectory()
        self.dirs.append(self.ipydir)
        self.dirs.append(self.nbdir)
        os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir1', u'sub ∂ir 1a')))
        os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir2', u'sub ∂ir 1b')))

        # start the ipython notebook, so we get the port number
        self.server_port = 0
        self._init_server()
        if self.server_port:
            self.cmd.append("--port=%i" % self.server_port)
        else:
            # don't launch tests if the server didn't start
            self.cmd = [sys.executable, '-c', 'raise SystemExit(1)']

    def print_extra_info(self):
        print("Running tests with notebook directory %r" % self.nbdir.name)

    @property
    def will_run(self):
        return all(have[a] for a in ['zmq', 'tornado', 'jinja2', 'casperjs', 'sqlite3'])

    def _init_server(self):
        "Start the notebook server in a separate process"
        self.server_command = command = [sys.executable,
            '-m', 'IPython.html',
            '--no-browser',
            '--ipython-dir', self.ipydir.name,
            '--notebook-dir', self.nbdir.name,
        ]
        # ipc doesn't work on Windows, and darwin has crazy-long temp paths,
        # which run afoul of ipc's maximum path length.
        if sys.platform.startswith('linux'):
            command.append('--KernelManager.transport=ipc')
        self.stream_capturer = c = StreamCapturer()
        c.start()
        self.server = subprocess.Popen(command, stdout=c.writefd, stderr=subprocess.STDOUT)
        self.server_info_file = os.path.join(self.ipydir.name,
            'profile_default', 'security', 'nbserver-%i.json' % self.server.pid
        )
        self._wait_for_server()

    def _wait_for_server(self):
        """Wait 30 seconds for the notebook server to start"""
        for i in range(300):
            if self.server.poll() is not None:
                return self._failed_to_start()
            if os.path.exists(self.server_info_file):
                self._load_server_info()
                return
            time.sleep(0.1)
        print("Notebook server-info file never arrived: %s" % self.server_info_file,
            file=sys.stderr
        )

    def _failed_to_start(self):
        """Notebook server exited prematurely"""
        captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
        print("Notebook failed to start: ", file=sys.stderr)
        print(self.server_command)
        print(captured, file=sys.stderr)

    def _load_server_info(self):
        """Notebook server started, load connection info from JSON"""
        with open(self.server_info_file) as f:
            info = json.load(f)
        self.server_port = info['port']

    def cleanup(self):
        try:
            self.server.terminate()
        except OSError:
            # already dead
            pass
        self.server.wait()
        self.stream_capturer.halt()
        TestController.cleanup(self)

def prepare_controllers(options):
    """Returns two lists of TestController instances, those to run, and those
    not to run."""
    testgroups = options.testgroups

    if testgroups:
        py_testgroups = [g for g in testgroups if (g in py_test_group_names) \
                                                or g.startswith('IPython.')]
        if 'js' in testgroups:
            js_testgroups = all_js_groups()
        else:
            js_testgroups = [g for g in testgroups if g not in py_testgroups]
    else:
        py_testgroups = py_test_group_names
        js_testgroups = all_js_groups()
        if not options.all:
            test_sections['parallel'].enabled = False

    c_js = [JSController(name) for name in js_testgroups]
    c_py = [PyTestController(name, options) for name in py_testgroups]

    controllers = c_py + c_js
    to_run = [c for c in controllers if c.will_run]
    not_run = [c for c in controllers if not c.will_run]
    return to_run, not_run

def do_run(controller, buffer_output=True):
    """Setup and run a test controller.

    If buffer_output is True, no output is displayed, to avoid it appearing
    interleaved. In this case, the caller is responsible for displaying test
    output on failure.

    Returns
    -------
    controller : TestController
      The same controller as passed in, as a convenience for using map() type
      APIs.
    exitcode : int
      The exit code of the test subprocess. Non-zero indicates failure.
    """
    try:
        try:
            controller.setup()
            if not buffer_output:
                controller.print_extra_info()
            controller.launch(buffer_output=buffer_output)
        except Exception:
            import traceback
            traceback.print_exc()
            return controller, 1  # signal failure

        exitcode = controller.wait()
        return controller, exitcode

    except KeyboardInterrupt:
        return controller, -signal.SIGINT
    finally:
        controller.cleanup()

def report():
    """Return a string with a summary report of test-related variables."""
    inf = get_sys_info()
    out = []
    def _add(name, value):
        out.append((name, value))

    _add('IPython version', inf['ipython_version'])
    _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
    _add('IPython package', compress_user(inf['ipython_path']))
    _add('Python version', inf['sys_version'].replace('\n',''))
    _add('sys.executable', compress_user(inf['sys_executable']))
    _add('Platform', inf['platform'])

    width = max(len(n) for (n,v) in out)
    out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]

    avail = []
    not_avail = []

    for k, is_avail in have.items():
        if is_avail:
            avail.append(k)
        else:
            not_avail.append(k)

    if avail:
        out.append('\nTools and libraries available at test time:\n')
        avail.sort()
        out.append('   ' + ' '.join(avail)+'\n')

    if not_avail:
        out.append('\nTools and libraries NOT available at test time:\n')
        not_avail.sort()
        out.append('   ' + ' '.join(not_avail)+'\n')

    return ''.join(out)

def run_iptestall(options):
    """Run the entire IPython test suite by calling nose and trial.

    This function constructs :class:`IPTester` instances for all IPython
    modules and package and then runs each of them.  This causes the modules
    and packages of IPython to be tested each in their own subprocess using
    nose.

    Parameters
    ----------

    All parameters are passed as attributes of the options object.

    testgroups : list of str
      Run only these sections of the test suite. If empty, run all the available
      sections.

    fast : int or None
      Run the test suite in parallel, using n simultaneous processes. If None
      is passed, one process is used per CPU core. Default 1 (i.e. sequential)

    inc_slow : bool
      Include slow tests, like IPython.parallel. By default, these tests aren't
      run.

    xunit : bool
      Produce Xunit XML output. This is written to multiple foo.xunit.xml files.

    coverage : bool or str
      Measure code coverage from tests. True will store the raw coverage data,
      or pass 'html' or 'xml' to get reports.

    extra_args : list
      Extra arguments to pass to the test subprocesses, e.g. '-v'
    """
    to_run, not_run = prepare_controllers(options)

    def justify(ltext, rtext, width=70, fill='-'):
        ltext += ' '
        rtext = (' ' + rtext).rjust(width - len(ltext), fill)
        return ltext + rtext

    # Run all test runners, tracking execution time
    failed = []
    t_start = time.time()

    print()
    if options.fast == 1:
        # This actually means sequential, i.e. with 1 job
        for controller in to_run:
            print('Test group:', controller.section)
            sys.stdout.flush()  # Show in correct order when output is piped
            controller, res = do_run(controller, buffer_output=False)
            if res:
                failed.append(controller)
                if res == -signal.SIGINT:
                    print("Interrupted")
                    break
            print()

    else:
        # Run tests concurrently
        try:
            pool = multiprocessing.pool.ThreadPool(options.fast)
            for (controller, res) in pool.imap_unordered(do_run, to_run):
                res_string = 'OK' if res == 0 else 'FAILED'
                print(justify('Test group: ' + controller.section, res_string))
                if res:
                    controller.print_extra_info()
                    print(bytes_to_str(controller.stdout))
                    failed.append(controller)
                    if res == -signal.SIGINT:
                        print("Interrupted")
                        break
        except KeyboardInterrupt:
            return

    for controller in not_run:
        print(justify('Test group: ' + controller.section, 'NOT RUN'))

    t_end = time.time()
    t_tests = t_end - t_start
    nrunners = len(to_run)
    nfail = len(failed)
    # summarize results
    print('_'*70)
    print('Test suite completed for system with the following information:')
    print(report())
    took = "Took %.3fs." % t_tests
    print('Status: ', end='')
    if not failed:
        print('OK (%d test groups).' % nrunners, took)
    else:
        # If anything went wrong, point out what command to rerun manually to
        # see the actual errors and individual summary
        failed_sections = [c.section for c in failed]
        print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
                                  nrunners, ', '.join(failed_sections)), took)
        print()
        print('You may wish to rerun these, with:')
        print('  iptest', *failed_sections)
        print()

    if options.coverage:
        from coverage import coverage
        cov = coverage(data_file='.coverage')
        cov.combine()
        cov.save()

        # Coverage HTML report
        if options.coverage == 'html':
            html_dir = 'ipy_htmlcov'
            shutil.rmtree(html_dir, ignore_errors=True)
            print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
            sys.stdout.flush()

            # Custom HTML reporter to clean up module names.
            from coverage.html import HtmlReporter
            class CustomHtmlReporter(HtmlReporter):
                def find_code_units(self, morfs):
                    super(CustomHtmlReporter, self).find_code_units(morfs)
                    for cu in self.code_units:
                        nameparts = cu.name.split(os.sep)
                        if 'IPython' not in nameparts:
                            continue
                        ix = nameparts.index('IPython')
                        cu.name = '.'.join(nameparts[ix:])

            # Reimplement the html_report method with our custom reporter
            cov._harvest_data()
            cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
                                 html_title='IPython test coverage',
                                )
            reporter = CustomHtmlReporter(cov, cov.config)
            reporter.report(None)
            print('done.')

        # Coverage XML report
        elif options.coverage == 'xml':
            cov.xml_report(outfile='ipy_coverage.xml')

    if failed:
        # Ensure that our exit code indicates failure
        sys.exit(1)

argparser = argparse.ArgumentParser(description='Run IPython test suite')
argparser.add_argument('testgroups', nargs='*',
                    help='Run specified groups of tests. If omitted, run '
                    'all tests.')
argparser.add_argument('--all', action='store_true',
                    help='Include slow tests not run by default.')
argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
                    help='Run test sections in parallel. This starts as many '
                    'processes as you have cores, or you can specify a number.')
argparser.add_argument('--xunit', action='store_true',
                    help='Produce Xunit XML results')
argparser.add_argument('--coverage', nargs='?', const=True, default=False,
                    help="Measure test coverage. Specify 'html' or "
                    "'xml' to get reports.")
argparser.add_argument('--subproc-streams', default='capture',
                    help="What to do with stdout/stderr from subprocesses. "
                    "'capture' (default), 'show' and 'discard' are the options.")

def default_options():
    """Get an argparse Namespace object with the default arguments, to pass to
    :func:`run_iptestall`.
    """
    options = argparser.parse_args([])
    options.extra_args = []
    return options

def main():
    # iptest doesn't work correctly if the working directory is the
    # root of the IPython source tree. Tell the user to avoid
    # frustration.
    if os.path.exists(os.path.join(os.getcwd(),
                                   'IPython', 'testing', '__main__.py')):
        print("Don't run iptest from the IPython source directory",
              file=sys.stderr)
        sys.exit(1)
    # Arguments after -- should be passed through to nose. Argparse treats
    # everything after -- as regular positional arguments, so we separate them
    # first.
    try:
        ix = sys.argv.index('--')
    except ValueError:
        to_parse = sys.argv[1:]
        extra_args = []
    else:
        to_parse = sys.argv[1:ix]
        extra_args = sys.argv[ix+1:]

    options = argparser.parse_args(to_parse)
    options.extra_args = extra_args

    run_iptestall(options)

if __name__ == '__main__':
    main()