# test_singlecomp.py ---
#
# Filename: test_singlecomp.py
# Description:
# Author: Subhasis Ray
# Maintainer:
# Created: Tue Jul 17 21:01:14 2012 (+0530)
# Version:
# Last-Updated: Sun Jun 25 15:37:21 2017 (-0400)
#           By: subha
#     Update #: 320
# URL:
# Keywords:
# Compatibility:
#
#

# Commentary:
#
# Test the ion channels with a single compartment.
#
#

# Change log:
#
# 2012-07-17 22:22:23 (+0530) Tested NaF2 and NaPF_SS against neuron
# test case.
#
#

# Code:

import os
os.environ['NUMPTHREADS'] = '1'
import uuid
import unittest
from datetime import datetime
import sys
sys.path.append('../../../python')

import numpy as np
from matplotlib import pyplot as plt

import moose
from testutils import *
from nachans import *
from kchans import *
from archan import *
from cachans import *
from capool import *

simdt = 0.25e-4
plotdt = 0.25e-4
simtime = 350e-3

erev = {
    'K': -100e-3,
    'Na': 50e-3,
    'Ca': 125e-3,
    'AR': -40e-3
    }

channel_density = {
    'NaF2':     1500.0,
    'NaPF_SS':  1.5,
    'KDR_FS':   1000.0,
    'KC_FAST':  100.0,
    'KA':       300.0,
    'KM':       37.5,
    'K2':       1.0,
    'KAHP_SLOWER':      1.0,
    'CaL':      5.0,
    'CaT_A':    1.0,
    'AR':       2.5
}

compartment_propeties = {
    'length': 20e-6,
    'diameter': 2e-6 * 7.5,
    'initVm': -65e-3,
    'Em': -65e-3,
    'Rm': 5.0,
    'Cm': 9e-3,
    'Ra': 1.0,
    'specific': True}

stimulus = [[100e-3, 50e-3, 3e-10], # delay[0], width[0], level[0]
            [1e9, 0, 0]]

def create_compartment(path, length, diameter, initVm, Em, Rm, Cm, Ra, specific=False):
    comp = moose.Compartment(path)
    comp.length = length
    comp.diameter = diameter
    comp.initVm = initVm
    comp.Em = Em
    if not specific:
        comp.Rm = Rm
        comp.Cm = Cm
        comp.Ra = Ra
    else:
        sarea = np.pi * length * diameter
        comp.Rm = Rm / sarea
        comp.Cm = Cm * sarea
        comp.Ra = 4.0 * Ra * length / (np.pi * diameter * diameter)
    return comp

def insert_channel(compartment, channeclass, gbar, density=False):
    channel = moose.copy(channeclass.prototype, compartment)[0]
    if not density:
        channel.Gbar = gbar
    else:
        channel.Gbar = gbar * np.pi * compartment.length * compartment.diameter
    moose.connect(channel, 'channel', compartment, 'channel')
    return channel

def insert_ca(compartment, phi, tau):
    ca = moose.copy(CaPool.prototype, compartment)[0]
    ca.B = phi / (np.pi * compartment.length * compartment.diameter)
    ca.tau = tau
    print( ca.path, ca.B, ca.tau)
    for chan in moose.wildcardFind('%s/#[TYPE=HHChannel]' % (compartment.path)):
        if chan.name.startswith('KC') or chan.name.startswith('KAHP'):
            moose.connect(ca, 'concOut', chan, 'concen')
        elif chan.name.startswith('CaL'):
            moose.connect(chan, 'IkOut', ca, 'current')
        else:
            continue
        moose.showfield(chan)
    return ca

class TestSingleComp(unittest.TestCase):
    def setUp(self):
        self.testId = uuid.uuid4().int
        self.container = moose.Neutral('test%d' % (self.testId))
        self.model = moose.Neutral('%s/model' % (self.container.path))
        self.data = moose.Neutral('%s/data' % (self.container.path))
        self.soma = create_compartment('%s/soma' % (self.model.path),
                                       **compartment_propeties)
        self.tables = {}
        tab = moose.Table('%s/Vm' % (self.data.path))
        self.tables['Vm'] = tab
        moose.connect(tab, 'requestOut', self.soma, 'getVm')
        for channelname, conductance in list(channel_density.items()):
            chanclass = eval(channelname)
            channel = insert_channel(self.soma, chanclass, conductance, density=True)
            if issubclass(chanclass, KChannel):
                channel.Ek = erev['K']
            elif issubclass(chanclass, NaChannel):
                channel.Ek = erev['Na']
            elif issubclass(chanclass, CaChannel):
                channel.Ek = erev['Ca']
            elif issubclass(chanclass, AR):
                channel.Ek = erev['AR']
            tab = moose.Table('%s/%s' % (self.data.path, channelname))
            moose.connect(tab, 'requestOut', channel, 'getGk')
            self.tables['Gk_'+channel.name] = tab
        archan = moose.HHChannel(self.soma.path + '/AR')
        archan.X = 0.0
        ca = insert_ca(self.soma, 2.6e7, 50e-3)
        tab = moose.Table('%s/Ca' % (self.data.path))
        self.tables['Ca'] = tab
        moose.connect(tab, 'requestOut', ca, 'getCa')
        self.pulsegen = moose.PulseGen('%s/inject' % (self.model.path))
        moose.connect(self.pulsegen, 'output', self.soma, 'injectMsg')
        tab = moose.Table('%s/injection' % (self.data.path))
        moose.connect(tab, 'requestOut', self.pulsegen, 'getOutputValue')
        self.tables['pulsegen'] = tab
        self.pulsegen.count = len(stimulus)
        for ii in range(len(stimulus)):
            self.pulsegen.delay[ii] = stimulus[ii][0]
            self.pulsegen.width[ii] = stimulus[ii][1]
            self.pulsegen.level[ii] = stimulus[ii][2]
        setup_clocks(simdt, plotdt)
        assign_clocks(self.model, self.data)
        moose.reinit()
        start = datetime.now()
        moose.start(simtime)
        end = datetime.now()
        delta = end - start
        print( 'Simulation of %g s finished in %g s' % (simtime, delta.seconds + delta.microseconds*1e-6))

    def testDefault(self):
        vm_axis = plt.subplot(2,1,1)
        ca_axis = plt.subplot(2,1,2)
        try:
            fname = os.path.join(config.mydir, 'nrn', 'data', 'singlecomp_Vm.dat')
            nrndata = np.loadtxt(fname)
            vm_axis.plot(nrndata[:,0], nrndata[:,1], label='Vm (mV) - nrn')
            ca_axis.plot(nrndata[:,0], nrndata[:,2], label='Ca (mM) - nrn')
        except IOError as e:
            print(e)
        tseries = np.linspace(0, simtime, len(self.tables['Vm'].vector)) * 1e3
        # plotcount = len(channel_density) + 1
        # rows = int(np.sqrt(plotcount) + 0.5)
        # columns = int(plotcount * 1.0/rows + 0.5)
        # print plotcount, rows, columns
        # plt.subplot(rows, columns, 1)
        vm_axis.plot(tseries, self.tables['Vm'].vector * 1e3, label='Vm (mV) - moose')
        vm_axis.plot(tseries, self.tables['pulsegen'].vector * 1e12, label='inject (pA)')
        ca_axis.plot(tseries, self.tables['Ca'].vector, label='Ca (mM) - moose')
        vm_axis.legend()
        ca_axis.legend()
        # ii = 2
        # for key, value in self.tables.items():
        #     if key.startswith('Gk'):
        #         plt.subplot(rows, columns, ii)
        #         plt.plot(tseries, value.vector, label=key)
        #         ii += 1
        #         plt.legend()
        plt.show()
        data = np.vstack((tseries*1e-3,
                          self.tables['Vm'].vector,
                          self.tables['Ca'].vector))
        np.savetxt(os.path.join(config.data_dir, 'singlecomp_Vm.dat'),
                   np.transpose(data))

if __name__ == '__main__':
    unittest.main()

#
# test_singlecomp.py ends here


# Mantid Repository : https://github.com/mantidproject/mantid
#
# Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
#   NScD Oak Ridge National Laboratory, European Spallation Source,
#   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
# SPDX - License - Identifier: GPL - 3.0 +
from qtpy.QtWidgets import (QApplication)  # noqa
from qtpy import QtCore, QtGui
import matplotlib
import sys
import os
try:
    from mantid import __version__ as __mtd_version
    from mantid import _bindir as __mtd_bin_dir
    # convert to major.minor
    __mtd_version = '.'.join(__mtd_version.split(".")[:2])
except ImportError:  # mantid not found
    __mtd_version = ''
    __mtd_bin_dir=''

def set_matplotlib_backend():
    '''MUST be called before anything tries to use matplotlib

    This will set the backend if it hasn't been already. It also returns
    the name of the backend to be the name to be used for importing the
    correct matplotlib widgets.'''
    backend = matplotlib.get_backend()
    if backend.startswith('module://'):
        if backend.endswith('qt4agg'):
            backend = 'Qt4Agg'
        elif backend.endswith('workbench') or backend.endswith('qt5agg'):
            backend = 'Qt5Agg'
    else:
        from qtpy import PYQT4, PYQT5  # noqa
        if PYQT5:
            backend = 'Qt5Agg'
        elif PYQT4:
            backend = 'Qt4Agg'
        else:
            raise RuntimeError('Do not know which matplotlib backend to set')
        matplotlib.use(backend)
    return backend

def get_qapplication():
    ''' Example usage:

    app, within_mantid = get_qapplication()
    reducer = eventFilterGUI.MainWindow()  # the main ui class in this file
    reducer.show()
    if not within_mantid:
        sys.exit(app.exec_())'''
    app = QApplication.instance()
    if app:
        return app, app.applicationName().lower().startswith('mantid')
    else:
        return QApplication(sys.argv), False

def __to_external_url(interface_name: str, section: str, external_url: str) -> QtCore.QUrl:
    if not external_url:
        template = 'http://docs.mantidproject.org/nightly/interfaces/{}/{}.html'
        external_url = template.format(section, interface_name)

    return QtCore.QUrl(external_url)

def __to_qthelp_url(interface_name: str, section: str, qt_url: str) -> str:
    if qt_url:
        return qt_url
    else:
        template = 'qthelp://org.sphinx.mantidproject.{}/doc/interfaces/{}/{}.html'
        return template.format(__mtd_version, section, interface_name)

def __get_collection_file(collection_file: str) -> str:
    if not collection_file:
        if not __mtd_bin_dir:
            return 'HELP COLLECTION FILE NOT FOUND'
        else:
            collection_file = os.path.join(__mtd_bin_dir, '../docs/qthelp/MantidProject.qhc')

    return os.path.abspath(collection_file)

def show_interface_help(mantidplot_name, assistant_process, area: str='',
                        collection_file: str='',
                        qt_url: str='', external_url: str=""):
    ''' Shows the help page for a custom interface
    @param mantidplot_name: used by showCustomInterfaceHelp
    @param assistant_process: needs to be started/closed from outside (see example below)
    @param collection_file: qth file containing the help in format used by qtassistant. The default is
    ``mantid._bindir + '../docs/qthelp/MantidProject.qhc'``
    @param qt_url: location of the help in the qth file. The default value is
    ``qthelp://org.sphinx.mantidproject.{mtdversion}/doc/interfaces/{mantidplot_name}.html``.
    @param external_url: location of external page to be displayed in the default browser. The default value is
    ``http://docs.mantidproject.org/nightly/interfaces/framework/{mantidplot_name}.html``

    Example using defaults:

    #in the __init__ function of the GUI add:
    self.assistant_process = QtCore.QProcess(self)
    self.mantidplot_name='DGS Planner'

    #add a help function in the GUI
    def help(self):
       show_interface_help(self.mantidplot_name,
                           self.assistant_process)

    #make sure you close the qtassistant when the GUI is closed
    def closeEvent(self, event):
        self.assistant_process.close()
        self.assistant_process.waitForFinished()
        event.accept()
    '''
    try:
        # try using built-in help in mantid
        import mantidqt
        mantidqt.interfacemanager.InterfaceManager().showCustomInterfaceHelp(mantidplot_name, area)
    except: #(ImportError, ModuleNotFoundError) raises the wrong type of error
        # built-in help failed, try external qtassistant then give up and launch a browser

        # cleanup previous version
        assistant_process.close()
        assistant_process.waitForFinished()

        # where to expect qtassistant
        helpapp = QtCore.QLibraryInfo.location(QtCore.QLibraryInfo.BinariesPath) + QtCore.QDir.separator()
        helpapp += 'assistant'

        collection_file = __get_collection_file(collection_file)
        if os.path.isfile(helpapp) and os.path.isfile(collection_file):
            # try to find the collection file and launch qtassistant
            args = ['-enableRemoteControl',
                    '-collectionFile', collection_file,
                    '-showUrl', __to_qthelp_url(mantidplot_name, area, qt_url)]

            assistant_process.close()
            assistant_process.waitForFinished()
            assistant_process.start(helpapp, args)
        else:
            # give up and upen a URL in default browser
            openUrl=QtGui.QDesktopServices.openUrl
            sysenv=QtCore.QProcessEnvironment.systemEnvironment()
            ldp=sysenv.value('LD_PRELOAD')
            if ldp:
                del os.environ['LD_PRELOAD']

            # create a url to the help in the default location
            openUrl(__to_external_url(mantidplot_name, area, external_url))

            if ldp:
                os.environ['LD_PRELOAD']=ldp


"""Metrics to assess performance on regression task

Functions named as ``*_score`` return a scalar value to maximize: the higher
the better

Function named as ``*_error`` or ``*_loss`` return a scalar value to minimize:
the lower the better
"""

# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Arnaud Joly <a.joly@ulg.ac.be>
#          Jochen Wersdorfer <jochen@wersdoerfer.de>
#          Lars Buitinck
#          Joel Nothman <joel.nothman@gmail.com>
#          Karan Desai <karandesai281196@gmail.com>
#          Noel Dawe <noel@dawe.me>
#          Manoj Kumar <manojkumarsivaraj334@gmail.com>
#          Michael Eickenberg <michael.eickenberg@gmail.com>
#          Konstantin Shmelkov <konstantin.shmelkov@polytechnique.edu>
# License: BSD 3 clause

from __future__ import division

import numpy as np

from ..utils.validation import check_array, check_consistent_length
from ..utils.validation import column_or_1d
from ..externals.six import string_types

__ALL__ = [
    "mean_absolute_error",
    "mean_squared_error",
    "mean_squared_log_error",
    "median_absolute_error",
    "r2_score",
    "explained_variance_score"
]

def _check_reg_targets(y_true, y_pred, multioutput):
    """Check that y_true and y_pred belong to the same regression task

    Parameters
    ----------
    y_true : array-like,

    y_pred : array-like,

    multioutput : array-like or string in ['raw_values', uniform_average',
        'variance_weighted'] or None
        None is accepted due to backward compatibility of r2_score().

    Returns
    -------
    type_true : one of {'continuous', continuous-multioutput'}
        The type of the true target data, as output by
        'utils.multiclass.type_of_target'

    y_true : array-like of shape = (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples, n_outputs)
        Estimated target values.

    multioutput : array-like of shape = (n_outputs) or string in ['raw_values',
        uniform_average', 'variance_weighted'] or None
        Custom output weights if ``multioutput`` is array-like or
        just the corresponding argument if ``multioutput`` is a
        correct keyword.

    """
    check_consistent_length(y_true, y_pred)
    y_true = check_array(y_true, ensure_2d=False)
    y_pred = check_array(y_pred, ensure_2d=False)

    if y_true.ndim == 1:
        y_true = y_true.reshape((-1, 1))

    if y_pred.ndim == 1:
        y_pred = y_pred.reshape((-1, 1))

    if y_true.shape[1] != y_pred.shape[1]:
        raise ValueError("y_true and y_pred have different number of output "
                         "({0}!={1})".format(y_true.shape[1], y_pred.shape[1]))

    n_outputs = y_true.shape[1]
    allowed_multioutput_str = ('raw_values', 'uniform_average',
                               'variance_weighted')
    if isinstance(multioutput, string_types):
        if multioutput not in allowed_multioutput_str:
            raise ValueError("Allowed 'multioutput' string values are {}. "
                             "You provided multioutput={!r}".format(
                                 allowed_multioutput_str,
                                 multioutput))
    elif multioutput is not None:
        multioutput = check_array(multioutput, ensure_2d=False)
        if n_outputs == 1:
            raise ValueError("Custom weights are useful only in "
                             "multi-output cases.")
        elif n_outputs != len(multioutput):
            raise ValueError(("There must be equally many custom weights "
                              "(%d) as outputs (%d).") %
                             (len(multioutput), n_outputs))
    y_type = 'continuous' if n_outputs == 1 else 'continuous-multioutput'

    return y_type, y_true, y_pred, multioutput

def mean_absolute_error(y_true, y_pred,
                        sample_weight=None,
                        multioutput='uniform_average'):
    """Mean absolute error regression loss

    Read more in the :ref:`User Guide <mean_absolute_error>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average']
        or array-like of shape (n_outputs)
        Defines aggregating of multiple output values.
        Array-like value defines weights used to average errors.

        'raw_values' :
            Returns a full set of errors in case of multioutput input.

        'uniform_average' :
            Errors of all outputs are averaged with uniform weight.

    Returns
    -------
    loss : float or ndarray of floats
        If multioutput is 'raw_values', then mean absolute error is returned
        for each output separately.
        If multioutput is 'uniform_average' or an ndarray of weights, then the
        weighted average of all output errors is returned.

        MAE output is non-negative floating point. The best value is 0.0.

    Examples
    --------
    >>> from sklearn.metrics import mean_absolute_error
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> mean_absolute_error(y_true, y_pred)
    0.5
    >>> y_true = [[0.5, 1], [-1, 1], [7, -6]]
    >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
    >>> mean_absolute_error(y_true, y_pred)
    0.75
    >>> mean_absolute_error(y_true, y_pred, multioutput='raw_values')
    array([ 0.5,  1. ])
    >>> mean_absolute_error(y_true, y_pred, multioutput=[0.3, 0.7])
    ... # doctest: +ELLIPSIS
    0.849...
    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)
    output_errors = np.average(np.abs(y_pred - y_true),
                               weights=sample_weight, axis=0)
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            return output_errors
        elif multioutput == 'uniform_average':
            # pass None as weights to np.average: uniform mean
            multioutput = None

    return np.average(output_errors, weights=multioutput)

def mean_squared_error(y_true, y_pred,
                       sample_weight=None,
                       multioutput='uniform_average'):
    """Mean squared error regression loss

    Read more in the :ref:`User Guide <mean_squared_error>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average']
        or array-like of shape (n_outputs)
        Defines aggregating of multiple output values.
        Array-like value defines weights used to average errors.

        'raw_values' :
            Returns a full set of errors in case of multioutput input.

        'uniform_average' :
            Errors of all outputs are averaged with uniform weight.

    Returns
    -------
    loss : float or ndarray of floats
        A non-negative floating point value (the best value is 0.0), or an
        array of floating point values, one for each individual target.

    Examples
    --------
    >>> from sklearn.metrics import mean_squared_error
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> mean_squared_error(y_true, y_pred)
    0.375
    >>> y_true = [[0.5, 1],[-1, 1],[7, -6]]
    >>> y_pred = [[0, 2],[-1, 2],[8, -5]]
    >>> mean_squared_error(y_true, y_pred)  # doctest: +ELLIPSIS
    0.708...
    >>> mean_squared_error(y_true, y_pred, multioutput='raw_values')
    ... # doctest: +ELLIPSIS
    array([ 0.416...,  1.        ])
    >>> mean_squared_error(y_true, y_pred, multioutput=[0.3, 0.7])
    ... # doctest: +ELLIPSIS
    0.824...

    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)
    output_errors = np.average((y_true - y_pred) ** 2, axis=0,
                               weights=sample_weight)
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            return output_errors
        elif multioutput == 'uniform_average':
            # pass None as weights to np.average: uniform mean
            multioutput = None

    return np.average(output_errors, weights=multioutput)

def mean_squared_log_error(y_true, y_pred,
                           sample_weight=None,
                           multioutput='uniform_average'):
    """Mean squared logarithmic error regression loss

    Read more in the :ref:`User Guide <mean_squared_log_error>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average'] \
            or array-like of shape = (n_outputs)

        Defines aggregating of multiple output values.
        Array-like value defines weights used to average errors.

        'raw_values' :
            Returns a full set of errors when the input is of multioutput
            format.

        'uniform_average' :
            Errors of all outputs are averaged with uniform weight.

    Returns
    -------
    loss : float or ndarray of floats
        A non-negative floating point value (the best value is 0.0), or an
        array of floating point values, one for each individual target.

    Examples
    --------
    >>> from sklearn.metrics import mean_squared_log_error
    >>> y_true = [3, 5, 2.5, 7]
    >>> y_pred = [2.5, 5, 4, 8]
    >>> mean_squared_log_error(y_true, y_pred)  # doctest: +ELLIPSIS
    0.039...
    >>> y_true = [[0.5, 1], [1, 2], [7, 6]]
    >>> y_pred = [[0.5, 2], [1, 2.5], [8, 8]]
    >>> mean_squared_log_error(y_true, y_pred)  # doctest: +ELLIPSIS
    0.044...
    >>> mean_squared_log_error(y_true, y_pred, multioutput='raw_values')
    ... # doctest: +ELLIPSIS
    array([ 0.004...,  0.083...])
    >>> mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7])
    ... # doctest: +ELLIPSIS
    0.060...

    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)

    if not (y_true >= 0).all() and not (y_pred >= 0).all():
        raise ValueError("Mean Squared Logarithmic Error cannot be used when "
                         "targets contain negative values.")

    return mean_squared_error(np.log(y_true + 1), np.log(y_pred + 1),
                              sample_weight, multioutput)

def median_absolute_error(y_true, y_pred):
    """Median absolute error regression loss

    Read more in the :ref:`User Guide <median_absolute_error>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples)
        Estimated target values.

    Returns
    -------
    loss : float
        A positive floating point value (the best value is 0.0).

    Examples
    --------
    >>> from sklearn.metrics import median_absolute_error
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> median_absolute_error(y_true, y_pred)
    0.5

    """
    y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred,
                                                   'uniform_average')
    if y_type == 'continuous-multioutput':
        raise ValueError("Multioutput not supported in median_absolute_error")
    return np.median(np.abs(y_pred - y_true))

def explained_variance_score(y_true, y_pred,
                             sample_weight=None,
                             multioutput='uniform_average'):
    """Explained variance regression score function

    Best possible score is 1.0, lower values are worse.

    Read more in the :ref:`User Guide <explained_variance_score>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average', \
                'variance_weighted'] or array-like of shape (n_outputs)
        Defines aggregating of multiple output scores.
        Array-like value defines weights used to average scores.

        'raw_values' :
            Returns a full set of scores in case of multioutput input.

        'uniform_average' :
            Scores of all outputs are averaged with uniform weight.

        'variance_weighted' :
            Scores of all outputs are averaged, weighted by the variances
            of each individual output.

    Returns
    -------
    score : float or ndarray of floats
        The explained variance or ndarray if 'multioutput' is 'raw_values'.

    Notes
    -----
    This is not a symmetric function.

    Examples
    --------
    >>> from sklearn.metrics import explained_variance_score
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> explained_variance_score(y_true, y_pred)  # doctest: +ELLIPSIS
    0.957...
    >>> y_true = [[0.5, 1], [-1, 1], [7, -6]]
    >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
    >>> explained_variance_score(y_true, y_pred, multioutput='uniform_average')
    ... # doctest: +ELLIPSIS
    0.983...

    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)

    y_diff_avg = np.average(y_true - y_pred, weights=sample_weight, axis=0)
    numerator = np.average((y_true - y_pred - y_diff_avg) ** 2,
                           weights=sample_weight, axis=0)

    y_true_avg = np.average(y_true, weights=sample_weight, axis=0)
    denominator = np.average((y_true - y_true_avg) ** 2,
                             weights=sample_weight, axis=0)

    nonzero_numerator = numerator != 0
    nonzero_denominator = denominator != 0
    valid_score = nonzero_numerator & nonzero_denominator
    output_scores = np.ones(y_true.shape[1])

    output_scores[valid_score] = 1 - (numerator[valid_score] /
                                      denominator[valid_score])
    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            # return scores individually
            return output_scores
        elif multioutput == 'uniform_average':
            # passing to np.average() None as weights results is uniform mean
            avg_weights = None
        elif multioutput == 'variance_weighted':
            avg_weights = denominator
    else:
        avg_weights = multioutput

    return np.average(output_scores, weights=avg_weights)

def r2_score(y_true, y_pred, sample_weight=None,
             multioutput="uniform_average"):
    """R^2 (coefficient of determination) regression score function.

    Best possible score is 1.0 and it can be negative (because the
    model can be arbitrarily worse). A constant model that always
    predicts the expected value of y, disregarding the input features,
    would get a R^2 score of 0.0.

    Read more in the :ref:`User Guide <r2_score>`.

    Parameters
    ----------
    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Ground truth (correct) target values.

    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
        Estimated target values.

    sample_weight : array-like of shape = (n_samples), optional
        Sample weights.

    multioutput : string in ['raw_values', 'uniform_average', \
'variance_weighted'] or None or array-like of shape (n_outputs)

        Defines aggregating of multiple output scores.
        Array-like value defines weights used to average scores.
        Default is "uniform_average".

        'raw_values' :
            Returns a full set of scores in case of multioutput input.

        'uniform_average' :
            Scores of all outputs are averaged with uniform weight.

        'variance_weighted' :
            Scores of all outputs are averaged, weighted by the variances
            of each individual output.

        .. versionchanged:: 0.19
            Default value of multioutput is 'uniform_average'.

    Returns
    -------
    z : float or ndarray of floats
        The R^2 score or ndarray of scores if 'multioutput' is
        'raw_values'.

    Notes
    -----
    This is not a symmetric function.

    Unlike most other scores, R^2 score may be negative (it need not actually
    be the square of a quantity R).

    References
    ----------
    .. [1] `Wikipedia entry on the Coefficient of determination
            <https://en.wikipedia.org/wiki/Coefficient_of_determination>`_

    Examples
    --------
    >>> from sklearn.metrics import r2_score
    >>> y_true = [3, -0.5, 2, 7]
    >>> y_pred = [2.5, 0.0, 2, 8]
    >>> r2_score(y_true, y_pred)  # doctest: +ELLIPSIS
    0.948...
    >>> y_true = [[0.5, 1], [-1, 1], [7, -6]]
    >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
    >>> r2_score(y_true, y_pred, multioutput='variance_weighted')
    ... # doctest: +ELLIPSIS
    0.938...
    >>> y_true = [1,2,3]
    >>> y_pred = [1,2,3]
    >>> r2_score(y_true, y_pred)
    1.0
    >>> y_true = [1,2,3]
    >>> y_pred = [2,2,2]
    >>> r2_score(y_true, y_pred)
    0.0
    >>> y_true = [1,2,3]
    >>> y_pred = [3,2,1]
    >>> r2_score(y_true, y_pred)
    -3.0
    """
    y_type, y_true, y_pred, multioutput = _check_reg_targets(
        y_true, y_pred, multioutput)

    if sample_weight is not None:
        sample_weight = column_or_1d(sample_weight)
        weight = sample_weight[:, np.newaxis]
    else:
        weight = 1.

    numerator = (weight * (y_true - y_pred) ** 2).sum(axis=0,
                                                      dtype=np.float64)
    denominator = (weight * (y_true - np.average(
        y_true, axis=0, weights=sample_weight)) ** 2).sum(axis=0,
                                                          dtype=np.float64)
    nonzero_denominator = denominator != 0
    nonzero_numerator = numerator != 0
    valid_score = nonzero_denominator & nonzero_numerator
    output_scores = np.ones([y_true.shape[1]])
    output_scores[valid_score] = 1 - (numerator[valid_score] /
                                      denominator[valid_score])
    # arbitrary set to zero to avoid -inf scores, having a constant
    # y_true is not interesting for scoring a regression anyway
    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.
    if isinstance(multioutput, string_types):
        if multioutput == 'raw_values':
            # return scores individually
            return output_scores
        elif multioutput == 'uniform_average':
            # passing None as weights results is uniform mean
            avg_weights = None
        elif multioutput == 'variance_weighted':
            avg_weights = denominator
            # avoid fail on constant y or one-element arrays
            if not np.any(nonzero_denominator):
                if not np.any(nonzero_numerator):
                    return 1.0
                else:
                    return 0.0
    else:
        avg_weights = multioutput

    return np.average(output_scores, weights=avg_weights)


#!/usr/bin/python

import sys
import os
import numpy as np
import matplotlib
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import subprocess
import traceback

pci_dev ={
    "name"      : "",
    "loc"       : "",
    "class"     : "",
    "vender"    : "",
    "device"    : "",
    "vd"        : "",
    "isBridge"  : 1,
    "driver"    : ""
        }

def is_root():
    return os.geteuid() == 0

def get_pci_list():
    out = subprocess.Popen(['lspci', '-nm'],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()
    lspci_str = stdout.decode('ascii')

    pci_list = []
    pcis = lspci_str.split('\n')
    for each_pci in pcis:
        pci = {}
        __ =  each_pci.split(" ")
        if len(__) < 4:
            continue
        pci["loc"]      = __[0].replace('"', '')
        pci["vender"]   = __[2].replace('"', '')
        pci["device"]   = __[3].replace('"', '')
        pci["vd"]       = ":".join([pci["vender"], pci["device"]])
        out = subprocess.Popen(['lspci', '-s', '{}'.format(pci["loc"]), "-mvk"],
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT)
        stdout, stderr = out.communicate()
        ss = stdout.decode('ascii')
        for line in ss.split("\n"):
            if ': ' in line:
                k, v = line.split(": ")
                if k.strip() == "Class":
                    pci['class'] = v.strip().replace('"', '')
                elif k.strip() == "Vendor":
                    pci['vender'] = v.strip().replace('"', '')
                elif k.strip() == "Device" and ss.split("\n").index(line) > 0:
                    pci['device'] = v.strip().replace('"', '')
                elif k.strip() == "Driver":
                    pci['driver'] = v.strip().replace('"', '')
                else:
                    pass
            else:
                continue
        pci_list.append(pci)
    return pci_list

def print_mach_info(tsc_freq, tsc_overhead, loops):
    print("-------------------------------")
    print("   tsc_freq   : {}".format(tsc_freq))
    print(" tsc_overhead : {} clocks".format(tsc_overhead))
    print("    loops     : {}".format(loops))
    print("-------------------------------")

def clock2ns(clocks, tsc_freq):
    return int(clocks*1000000000/tsc_freq)

def plot_y(y, fname):
    num_width = 10
    ymin = int(min(y))-1
    ymax = int(max(y))+1

    print("Max. and Min. latencies are {}ns  {}ns".format(ymax, ymin))
    margin = max(num_width, 5)
    bins = [ii for ii in range(ymin-margin, ymax+margin, num_width)]

    plt.yscale('log')
    n, bins, patches = plt.hist(y, bins, range=(min(y), max(y)), width=10, color='blue')
    plt.xlabel('nanoseconds')
    plt.ylabel('Probability')
    plt.title('Histogram of PCIe latencies (%s samples)' % len(y))

    plt.savefig(fname, dpi=200, format='png')

def main():
    loops = 0
    if len(sys.argv) < 2:
        print("Usage: {}  [0000]:XX:XX.X [loops]".format(sys.argv[0]))
        exit(-1)
    else:
        pci_test = sys.argv[1]
        if pci_test.startswith('0000:'):
            pci_test = sys.argv[0][5:]
        if len(sys.argv) == 3:
            loops = int(sys.argv[2])
        else:
            loops = 100000

    ### must be root to run the script
    if not is_root():
        print("Need root privillege! run as root!")
        exit(-1)

    ### get all devices in this computer
    pcis = get_pci_list()
    if pci_test not in [pp['loc'] for pp in pcis]:
        print("existing PCI devices:")
        for __ in pcis:
            print(__)
        print("{} not found!".format(pci_test))
        exit(-1)

    for p in pcis:
        if p['loc'] == pci_test:
            pci_test = p
    unbind_file = "/sys/bus/pci/devices/0000\:{}/driver/unbind"
    unbind_file = unbind_file.format(pci_test['loc'].replace(':', '\:'))
    if os.path.exists(unbind_file):
        print("Unbind file {} not found!".format(unbind_file))
        exit(-1)
    unbind_ss = 'echo -n "0000:{}" > {}'.format(pci_test['loc'], unbind_file)
    os.system(unbind_ss)

    # insert module
    os.system("make")
    print("finished compiling the pcie-lat, insmod...");
    ins_command = "sudo insmod ./pcie-lat.ko ids={}".format(pci_test['vd'])
    print(ins_command)
    os.system(ins_command)

    # couting
    try:
        sys_path_head = "/sys/bus/pci/devices/0000:{}/pcie-lat/{}/pcielat_"
        sys_path_head = sys_path_head.format(pci_test['loc'], pci_test['loc'])
        tsc_freq        = 0
        tsc_overhead    = 0
        with open(sys_path_head+'tsc_freq', 'r') as __:
            tsc_freq = int(float(__.read()))
        with open(sys_path_head+'tsc_overhead', 'r') as __:
            tsc_overhead = int(float(__.read()))
        with open(sys_path_head+'loops', 'w') as __:
            __.write(str(loops))
        with open(sys_path_head+'target_bar', 'w') as __:
            __.write('0')
        print_mach_info(tsc_freq, tsc_overhead, loops)
        with open(sys_path_head+'measure', 'w') as __:
            __.write('0')
        with open('/dev/pcie-lat/{}'.format(pci_test['loc']), 'rb') as __:
            y = []
            cc = __.read(16)
            while cc:
                acc = 0
                acc2 = 0
                for ii in range(8):
                    acc = acc*256 + int(cc[7-ii])
                    acc2 = acc2*256 + int(cc[15-ii])
                y.append(clock2ns(acc2, tsc_freq))
                # read next
                cc = __.read(16)
            fname = "pcie_lat_loops{}_{}.png"
            fname = fname.format(loops, pci_test['loc'].replace(':', '..'))
            print("plot the graph")
            plot_y(y, fname)

    except Exception:
        traceback.print_exc()
        print("Removing module : sudo rmmod pcie-lat.ko")
        os.system("sudo rmmod pcie-lat.ko")
        exit(-1)

    # remove module
    print("Removing module : sudo rmmod pcie-lat.ko")
    os.system("sudo rmmod pcie-lat.ko")

if __name__ == "__main__":
    main()


#! /usr/bin/env python
from unittest import TestCase
from pandashells.lib.module_checker_lib import check_for_modules
from pandashells.lib import module_checker_lib
from mock import patch

class ModuleCheckerTests(TestCase):
    def setUp(self):
        module_checker_lib.CMD_DICT['fakemodule1'] = 'pip install fakemodule1'
        module_checker_lib.CMD_DICT['fakemodule2'] = 'pip install fakemodule2'
        module_checker_lib.CMD_DICT['os'] = 'part of standard module'

    def test_check_for_modules_unrecognized(self):
        """
        check_for_modules() raises error when module is unrecognized
        """
        with self.assertRaises(ValueError):
            check_for_modules(['not_a_module'])

    @patch('pandashells.lib.module_checker_lib.importlib.import_module')
    def test_check_for_modules_no_modules(self, import_module_mock):
        """
        check_for_modules() does nothing when module list is empty
        """
        check_for_modules([])
        self.assertFalse(import_module_mock.called)

    def test_check_for_modules_existing_module(self):
        """
        check_for_modules() successfully finds existing module
        """
        check_for_modules(['os'])

    def test_check_for_modules_bad(self):
        """
        check_for_modules() correctly identifies missing modules
        """
        with self.assertRaises(ImportError):
            check_for_modules(['fakemodule1', 'fakemodule2'])


""" Joblib is a set of tools to provide **lightweight pipelining in
Python**. In particular, joblib offers:

  1. transparent disk-caching of the output values and lazy re-evaluation
     (memoize pattern)

  2. easy simple parallel computing

  3. logging and tracing of the execution

Joblib is optimized to be **fast** and **robust** in particular on large
data and has specific optimizations for `numpy` arrays. It is
**BSD-licensed**.

    ============================== ============================================
    **User documentation**:        http://pythonhosted.org/joblib

    **Download packages**:         http://pypi.python.org/pypi/joblib#downloads

    **Source code**:               http://github.com/joblib/joblib

    **Report issues**:             http://github.com/joblib/joblib/issues
    ============================== ============================================

Vision
--------

The vision is to provide tools to easily achieve better performance and
reproducibility when working with long running jobs.

 *  **Avoid computing twice the same thing**: code is rerun over an
    over, for instance when prototyping computational-heavy jobs (as in
    scientific development), but hand-crafted solution to alleviate this
    issue is error-prone and often leads to unreproducible results

 *  **Persist to disk transparently**: persisting in an efficient way
    arbitrary objects containing large data is hard. Using
    joblib's caching mechanism avoids hand-written persistence and
    implicitly links the file on disk to the execution context of
    the original Python object. As a result, joblib's persistence is
    good for resuming an application status or computational job, eg
    after a crash.

Joblib strives to address these problems while **leaving your code and
your flow control as unmodified as possible** (no framework, no new
paradigms).

Main features
------------------

1) **Transparent and fast disk-caching of output value:** a memoize or
   make-like functionality for Python functions that works well for
   arbitrary Python objects, including very large numpy arrays. Separate
   persistence and flow-execution logic from domain logic or algorithmic
   code by writing the operations as a set of steps with well-defined
   inputs and  outputs: Python functions. Joblib can save their
   computation to disk and rerun it only if necessary::

      >>> import numpy as np
      >>> from sklearn.externals.joblib import Memory
      >>> mem = Memory(cachedir='/tmp/joblib')
      >>> import numpy as np
      >>> a = np.vander(np.arange(3)).astype(np.float)
      >>> square = mem.cache(np.square)
      >>> b = square(a)                                   # doctest: +ELLIPSIS
      ________________________________________________________________________________
      [Memory] Calling square...
      square(array([[ 0.,  0.,  1.],
             [ 1.,  1.,  1.],
             [ 4.,  2.,  1.]]))
      ___________________________________________________________square - 0...s, 0.0min

      >>> c = square(a)
      >>> # The above call did not trigger an evaluation

2) **Embarrassingly parallel helper:** to make is easy to write readable
   parallel code and debug it quickly::

      >>> from sklearn.externals.joblib import Parallel, delayed
      >>> from math import sqrt
      >>> Parallel(n_jobs=1)(delayed(sqrt)(i**2) for i in range(10))
      [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]

3) **Logging/tracing:** The different functionalities will
   progressively acquire better logging mechanism to help track what
   has been ran, and capture I/O easily. In addition, Joblib will
   provide a few I/O primitives, to easily define define logging and
   display streams, and provide a way of compiling a report.
   We want to be able to quickly inspect what has been run.

4) **Fast compressed Persistence**: a replacement for pickle to work
   efficiently on Python objects containing large data (
   *joblib.dump* & *joblib.load* ).

..
    >>> import shutil ; shutil.rmtree('/tmp/joblib/')

"""

# PEP0440 compatible formatted version, see:
# https://www.python.org/dev/peps/pep-0440/
#
# Generic release markers:
# X.Y
# X.Y.Z # For bugfix releases
#
# Admissible pre-release markers:
# X.YaN # Alpha release
# X.YbN # Beta release
# X.YrcN # Release Candidate
# X.Y # Final release
#
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
#
__version__ = '0.9.0b3'

from .memory import Memory, MemorizedResult
from .logger import PrintTime
from .logger import Logger
from .hashing import hash
from .numpy_pickle import dump
from .numpy_pickle import load
from .parallel import Parallel
from .parallel import delayed
from .parallel import cpu_count


"""Caching loader for the 20 newsgroups text classification dataset

The description of the dataset is available on the official website at:

    http://people.csail.mit.edu/jrennie/20Newsgroups/

Quoting the introduction:

    The 20 Newsgroups data set is a collection of approximately 20,000
    newsgroup documents, partitioned (nearly) evenly across 20 different
    newsgroups. To the best of my knowledge, it was originally collected
    by Ken Lang, probably for his Newsweeder: Learning to filter netnews
    paper, though he does not explicitly mention this collection. The 20
    newsgroups collection has become a popular data set for experiments
    in text applications of machine learning techniques, such as text
    classification and text clustering.

This dataset loader will download the recommended "by date" variant of the
dataset and which features a point in time split between the train and
test sets. The compressed dataset size is around 14 Mb compressed. Once
uncompressed the train set is 52 MB and the test set is 34 MB.

The data is downloaded, extracted and cached in the '~/scikit_learn_data'
folder.

The `fetch_20newsgroups` function will not vectorize the data into numpy
arrays but the dataset lists the filenames of the posts and their categories
as target labels.

The `fetch_20newsgroups_vectorized` function will in addition do a simple
tf-idf vectorization step.

"""
# Copyright (c) 2011 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

import os
import logging
import tarfile
import pickle
import shutil
import re
import codecs

import numpy as np
import scipy.sparse as sp

from .base import get_data_home
from .base import Bunch
from .base import load_files
from ..utils import check_random_state
from ..feature_extraction.text import CountVectorizer
from ..preprocessing import normalize
from ..externals import joblib, six

if six.PY3:
    from urllib.request import urlopen
else:
    from urllib2 import urlopen

logger = logging.getLogger(__name__)

URL = ("http://people.csail.mit.edu/jrennie/"
       "20Newsgroups/20news-bydate.tar.gz")
ARCHIVE_NAME = "20news-bydate.tar.gz"
CACHE_NAME = "20news-bydate.pkz"
TRAIN_FOLDER = "20news-bydate-train"
TEST_FOLDER = "20news-bydate-test"

def download_20newsgroups(target_dir, cache_path):
    """Download the 20 newsgroups data and stored it as a zipped pickle."""
    archive_path = os.path.join(target_dir, ARCHIVE_NAME)
    train_path = os.path.join(target_dir, TRAIN_FOLDER)
    test_path = os.path.join(target_dir, TEST_FOLDER)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    if os.path.exists(archive_path):
        # Download is not complete as the .tar.gz file is removed after
        # download.
        logger.warning("Download was incomplete, downloading again.")
        os.remove(archive_path)

    logger.warning("Downloading dataset from %s (14 MB)", URL)
    opener = urlopen(URL)
    with open(archive_path, 'wb') as f:
        f.write(opener.read())

    logger.info("Decompressing %s", archive_path)
    tarfile.open(archive_path, "r:gz").extractall(path=target_dir)
    os.remove(archive_path)

    # Store a zipped pickle
    cache = dict(train=load_files(train_path, encoding='latin1'),
                 test=load_files(test_path, encoding='latin1'))
    compressed_content = codecs.encode(pickle.dumps(cache), 'zlib_codec')
    with open(cache_path, 'wb') as f:
        f.write(compressed_content)

    shutil.rmtree(target_dir)
    return cache

def strip_newsgroup_header(text):
    """
    Given text in "news" format, strip the headers, by removing everything
    before the first blank line.
    """
    _before, _blankline, after = text.partition('\n\n')
    return after

_QUOTE_RE = re.compile(r'(writes in|writes:|wrote:|says:|said:'
                       r'|^In article|^Quoted from|^\||^>)')

def strip_newsgroup_quoting(text):
    """
    Given text in "news" format, strip lines beginning with the quote
    characters > or |, plus lines that often introduce a quoted section
    (for example, because they contain the string 'writes:'.)
    """
    good_lines = [line for line in text.split('\n')
                  if not _QUOTE_RE.search(line)]
    return '\n'.join(good_lines)

def strip_newsgroup_footer(text):
    """
    Given text in "news" format, attempt to remove a signature block.

    As a rough heuristic, we assume that signatures are set apart by either
    a blank line or a line made of hyphens, and that it is the last such line
    in the file (disregarding blank lines at the end).
    """
    lines = text.strip().split('\n')
    for line_num in range(len(lines) - 1, -1, -1):
        line = lines[line_num]
        if line.strip().strip('-') == '':
            break

    if line_num > 0:
        return '\n'.join(lines[:line_num])
    else:
        return text

def fetch_20newsgroups(data_home=None, subset='train', categories=None,
                       shuffle=True, random_state=42,
                       remove=(),
                       download_if_missing=True):
    """Load the filenames and data from the 20 newsgroups dataset.

    Read more in the :ref:`User Guide <20newsgroups>`.

    Parameters
    ----------
    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify a download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    categories: None or collection of string or unicode
        If None (default), load all the categories.
        If not None, list of category names to load (other categories
        ignored).

    shuffle: bool, optional
        Whether or not to shuffle the data: might be important for models that
        make the assumption that the samples are independent and identically
        distributed (i.i.d.), such as stochastic gradient descent.

    random_state: numpy random number generator or seed integer
        Used to shuffle the dataset.

    download_if_missing: optional, True by default
        If False, raise an IOError if the data is not locally available
        instead of trying to download the data from the source site.

    remove: tuple
        May contain any subset of ('headers', 'footers', 'quotes'). Each of
        these are kinds of text that will be detected and removed from the
        newsgroup posts, preventing classifiers from overfitting on
        metadata.

        'headers' removes newsgroup headers, 'footers' removes blocks at the
        ends of posts that look like signatures, and 'quotes' removes lines
        that appear to be quoting another post.

        'headers' follows an exact standard; the other filters are not always
        correct.
    """

    data_home = get_data_home(data_home=data_home)
    cache_path = os.path.join(data_home, CACHE_NAME)
    twenty_home = os.path.join(data_home, "20news_home")
    cache = None
    if os.path.exists(cache_path):
        try:
            with open(cache_path, 'rb') as f:
                compressed_content = f.read()
            uncompressed_content = codecs.decode(
                compressed_content, 'zlib_codec')
            cache = pickle.loads(uncompressed_content)
        except Exception as e:
            print(80 * '_')
            print('Cache loading failed')
            print(80 * '_')
            print(e)

    if cache is None:
        if download_if_missing:
            cache = download_20newsgroups(target_dir=twenty_home,
                                          cache_path=cache_path)
        else:
            raise IOError('20Newsgroups dataset not found')

    if subset in ('train', 'test'):
        data = cache[subset]
    elif subset == 'all':
        data_lst = list()
        target = list()
        filenames = list()
        for subset in ('train', 'test'):
            data = cache[subset]
            data_lst.extend(data.data)
            target.extend(data.target)
            filenames.extend(data.filenames)

        data.data = data_lst
        data.target = np.array(target)
        data.filenames = np.array(filenames)
    else:
        raise ValueError(
            "subset can only be 'train', 'test' or 'all', got '%s'" % subset)

    data.description = 'the 20 newsgroups by date dataset'

    if 'headers' in remove:
        data.data = [strip_newsgroup_header(text) for text in data.data]
    if 'footers' in remove:
        data.data = [strip_newsgroup_footer(text) for text in data.data]
    if 'quotes' in remove:
        data.data = [strip_newsgroup_quoting(text) for text in data.data]

    if categories is not None:
        labels = [(data.target_names.index(cat), cat) for cat in categories]
        # Sort the categories to have the ordering of the labels
        labels.sort()
        labels, categories = zip(*labels)
        mask = np.in1d(data.target, labels)
        data.filenames = data.filenames[mask]
        data.target = data.target[mask]
        # searchsorted to have continuous labels
        data.target = np.searchsorted(labels, data.target)
        data.target_names = list(categories)
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[mask]
        data.data = data_lst.tolist()

    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(data.target.shape[0])
        random_state.shuffle(indices)
        data.filenames = data.filenames[indices]
        data.target = data.target[indices]
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[indices]
        data.data = data_lst.tolist()

    return data

def fetch_20newsgroups_vectorized(subset="train", remove=(), data_home=None):
    """Load the 20 newsgroups dataset and transform it into tf-idf vectors.

    This is a convenience function; the tf-idf transformation is done using the
    default settings for `sklearn.feature_extraction.text.Vectorizer`. For more
    advanced usage (stopword filtering, n-gram extraction, etc.), combine
    fetch_20newsgroups with a custom `Vectorizer` or `CountVectorizer`.

    Read more in the :ref:`User Guide <20newsgroups>`.

    Parameters
    ----------

    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify an download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    remove: tuple
        May contain any subset of ('headers', 'footers', 'quotes'). Each of
        these are kinds of text that will be detected and removed from the
        newsgroup posts, preventing classifiers from overfitting on
        metadata.

        'headers' removes newsgroup headers, 'footers' removes blocks at the
        ends of posts that look like signatures, and 'quotes' removes lines
        that appear to be quoting another post.

    Returns
    -------

    bunch : Bunch object
        bunch.data: sparse matrix, shape [n_samples, n_features]
        bunch.target: array, shape [n_samples]
        bunch.target_names: list, length [n_classes]
    """
    data_home = get_data_home(data_home=data_home)
    filebase = '20newsgroup_vectorized'
    if remove:
        filebase += 'remove-' + ('-'.join(remove))
    target_file = os.path.join(data_home, filebase + ".pk")

    # we shuffle but use a fixed seed for the memoization
    data_train = fetch_20newsgroups(data_home=data_home,
                                    subset='train',
                                    categories=None,
                                    shuffle=True,
                                    random_state=12,
                                    remove=remove)

    data_test = fetch_20newsgroups(data_home=data_home,
                                   subset='test',
                                   categories=None,
                                   shuffle=True,
                                   random_state=12,
                                   remove=remove)

    if os.path.exists(target_file):
        X_train, X_test = joblib.load(target_file)
    else:
        vectorizer = CountVectorizer(dtype=np.int16)
        X_train = vectorizer.fit_transform(data_train.data).tocsr()
        X_test = vectorizer.transform(data_test.data).tocsr()
        joblib.dump((X_train, X_test), target_file, compress=9)

    # the data is stored as int16 for compactness
    # but normalize needs floats
    X_train = X_train.astype(np.float64)
    X_test = X_test.astype(np.float64)
    normalize(X_train, copy=False)
    normalize(X_test, copy=False)

    target_names = data_train.target_names

    if subset == "train":
        data = X_train
        target = data_train.target
    elif subset == "test":
        data = X_test
        target = data_test.target
    elif subset == "all":
        data = sp.vstack((X_train, X_test)).tocsr()
        target = np.concatenate((data_train.target, data_test.target))
    else:
        raise ValueError("%r is not a valid subset: should be one of "
                         "['train', 'test', 'all']" % subset)

    return Bunch(data=data, target=target, target_names=target_names)


'''VAR and VARMA process

this doesn't actually do much, trying out a version for a time loop

alternative representation:
* textbook, different blocks in matrices
* Kalman filter
* VAR, VARX and ARX could be calculated with signal.lfilter
  only tried some examples, not implemented

TODO: try minimizing sum of squares of (Y-Yhat)

Note: filter has smallest lag at end of array and largest lag at beginning,
    be careful for asymmetric lags coefficients
    check this again if it is consistently used

changes
2009-09-08 : separated from movstat.py

Author : josefpkt
License : BSD
'''

from __future__ import print_function
import numpy as np
from scipy import signal
#import matplotlib.pylab as plt
from numpy.testing import assert_array_equal, assert_array_almost_equal

#NOTE: this just returns that predicted values given the
#B matrix in polynomial form.
#TODO: make sure VAR class returns B/params in this form.
def VAR(x,B, const=0):
    ''' multivariate linear filter

    Parameters
    ----------
    x: (TxK) array
        columns are variables, rows are observations for time period
    B: (PxKxK) array
        b_t-1 is bottom "row", b_t-P is top "row" when printing
        B(:,:,0) is lag polynomial matrix for variable 1
        B(:,:,k) is lag polynomial matrix for variable k
        B(p,:,k) is pth lag for variable k
        B[p,:,:].T corresponds to A_p in Wikipedia
    const: float or array (not tested)
        constant added to autoregression

    Returns
    -------
    xhat: (TxK) array
        filtered, predicted values of x array

    Notes
    -----
    xhat(t,i) = sum{_p}sum{_k} { x(t-P:t,:) .* B(:,:,i) }  for all i = 0,K-1, for all t=p..T

    xhat does not include the forecasting observation, xhat(T+1),
    xhat is 1 row shorter than signal.correlate

    References
    ----------
    http://en.wikipedia.org/wiki/Vector_Autoregression
    http://en.wikipedia.org/wiki/General_matrix_notation_of_a_VAR(p)
    '''
    p = B.shape[0]
    T = x.shape[0]
    xhat = np.zeros(x.shape)
    for t in range(p,T): #[p+2]:#
##        print(p,T)
##        print(x[t-p:t,:,np.newaxis].shape)
##        print(B.shape)
        #print(x[t-p:t,:,np.newaxis])
        xhat[t,:] = const + (x[t-p:t,:,np.newaxis]*B).sum(axis=1).sum(axis=0)
    return xhat

def VARMA(x,B,C, const=0):
    ''' multivariate linear filter

    x (TxK)
    B (PxKxK)

    xhat(t,i) = sum{_p}sum{_k} { x(t-P:t,:) .* B(:,:,i) } +
                sum{_q}sum{_k} { e(t-Q:t,:) .* C(:,:,i) }for all i = 0,K-1

    '''
    P = B.shape[0]
    Q = C.shape[0]
    T = x.shape[0]
    xhat = np.zeros(x.shape)
    e = np.zeros(x.shape)
    start = max(P,Q)
    for t in range(start,T): #[p+2]:#
##        print(p,T
##        print(x[t-p:t,:,np.newaxis].shape
##        print(B.shape
        #print(x[t-p:t,:,np.newaxis]
        xhat[t,:] =  const + (x[t-P:t,:,np.newaxis]*B).sum(axis=1).sum(axis=0) + \
                     (e[t-Q:t,:,np.newaxis]*C).sum(axis=1).sum(axis=0)
        e[t,:] = x[t,:] - xhat[t,:]
    return xhat, e

if __name__ == '__main__':

    T = 20
    K = 2
    P = 3
    #x = np.arange(10).reshape(5,2)
    x = np.column_stack([np.arange(T)]*K)
    B = np.ones((P,K,K))
    #B[:,:,1] = 2
    B[:,:,1] = [[0,0],[0,0],[0,1]]
    xhat = VAR(x,B)
    print(np.all(xhat[P:,0]==np.correlate(x[:-1,0],np.ones(P))*2))
    #print(xhat)

    T = 20
    K = 2
    Q = 2
    P = 3
    const = 1
    #x = np.arange(10).reshape(5,2)
    x = np.column_stack([np.arange(T)]*K)
    B = np.ones((P,K,K))
    #B[:,:,1] = 2
    B[:,:,1] = [[0,0],[0,0],[0,1]]
    C = np.zeros((Q,K,K))
    xhat1 = VAR(x,B, const=const)
    xhat2, err2 = VARMA(x,B,C, const=const)
    print(np.all(xhat2 == xhat1))
    print(np.all(xhat2[P:,0] == np.correlate(x[:-1,0],np.ones(P))*2+const))

    C[1,1,1] = 0.5
    xhat3, err3 = VARMA(x,B,C)

    x = np.r_[np.zeros((P,K)),x]  #prepend inital conditions
    xhat4, err4 = VARMA(x,B,C)

    C[1,1,1] = 1
    B[:,:,1] = [[0,0],[0,0],[0,1]]
    xhat5, err5 = VARMA(x,B,C)
    #print(err5)

    #in differences
    #VARMA(np.diff(x,axis=0),B,C)

    #Note:
    # * signal correlate applies same filter to all columns if kernel.shape[1]<K
    #   e.g. signal.correlate(x0,np.ones((3,1)),'valid')
    # * if kernel.shape[1]==K, then `valid` produces a single column
    #   -> possible to run signal.correlate K times with different filters,
    #      see the following example, which replicates VAR filter
    x0 = np.column_stack([np.arange(T), 2*np.arange(T)])
    B[:,:,0] = np.ones((P,K))
    B[:,:,1] = np.ones((P,K))
    B[1,1,1] = 0
    xhat0 = VAR(x0,B)
    xcorr00 = signal.correlate(x0,B[:,:,0])#[:,0]
    xcorr01 = signal.correlate(x0,B[:,:,1])
    print(np.all(signal.correlate(x0,B[:,:,0],'valid')[:-1,0]==xhat0[P:,0]))
    print(np.all(signal.correlate(x0,B[:,:,1],'valid')[:-1,0]==xhat0[P:,1]))

    #import error
    #from movstat import acovf, acf
    from statsmodels.tsa.stattools import acovf, acf
    aav = acovf(x[:,0])
    print(aav[0] == np.var(x[:,0]))
    aac = acf(x[:,0])