import sys
import math

from PyQt4 import QtGui
from PyQt4.QtCore import Qt

import numpy as np
import sklearn.cross_validation as skl_cross_validation

from Orange.widgets import widget, gui
from Orange.widgets.settings import Setting
from Orange.data import Table
from Orange.data.sql.table import SqlTable

class OWDataSampler(widget.OWWidget):
    name = "Data Sampler"
    description = "Randomly draw a subset of data points " \
                  "from the input data set."
    icon = "icons/DataSampler.svg"
    priority = 100
    category = "Data"
    keywords = ["data", "sample"]
    inputs = [("Data", Table, "set_data")]
    outputs = [("Data Sample", Table, widget.Default),
               ("Remaining Data", Table)]

    want_main_area = False
    resizing_enabled = False

    RandomSeed = 42
    FixedProportion, FixedSize, CrossValidation, Bootstrap = range(4)
    SqlTime, SqlProportion = range(2)

    use_seed = Setting(False)
    replacement = Setting(False)
    stratify = Setting(False)
    sql_dl = Setting(False)
    sampling_type = Setting(FixedProportion)
    sampleSizeNumber = Setting(1)
    sampleSizePercentage = Setting(70)
    sampleSizeSqlTime = Setting(1)
    sampleSizeSqlPercentage = Setting(0.1)
    number_of_folds = Setting(10)
    selectedFold = Setting(1)

    def __init__(self):
        super().__init__()
        self.data = None
        self.indices = None
        self.sampled_instances = self.remaining_instances = None

        box = gui.widgetBox(self.controlArea, "Information")
        self.dataInfoLabel = gui.widgetLabel(box, 'No data on input.')
        self.outputInfoLabel = gui.widgetLabel(box, ' ')

        self.sampling_box = gui.widgetBox(self.controlArea, "Sampling Type")
        sampling = gui.radioButtons(self.sampling_box, self, "sampling_type",
                                    callback=self.sampling_type_changed)

        def set_sampling_type(i):
            def f():
                self.sampling_type = i
                self.sampling_type_changed()
            return f

        gui.appendRadioButton(sampling, "Fixed proportion of data:")
        self.sampleSizePercentageSlider = gui.hSlider(
            gui.indentedBox(sampling), self,
            "sampleSizePercentage",
            minValue=0, maxValue=99, ticks=10, labelFormat="%d %%",
            callback=set_sampling_type(self.FixedProportion),
            addSpace=12)

        gui.appendRadioButton(sampling, "Fixed sample size:")
        ibox = gui.indentedBox(sampling)
        self.sampleSizeSpin = gui.spin(
            ibox, self, "sampleSizeNumber", label="Instances: ",
            minv=1, maxv=2 ** 31 - 1,
            callback=set_sampling_type(self.FixedSize))
        gui.checkBox(
            ibox, self, "replacement", "Sample with replacement",
            callback=set_sampling_type(self.FixedSize),
            addSpace=12)

        gui.appendRadioButton(sampling, "Cross Validation:")
        form = QtGui.QFormLayout(
            formAlignment=Qt.AlignLeft | Qt.AlignTop,
            labelAlignment=Qt.AlignLeft,
            fieldGrowthPolicy=QtGui.QFormLayout.AllNonFixedFieldsGrow)
        ibox = gui.indentedBox(sampling, addSpace=True, orientation=form)
        form.addRow("Number of folds",
                    gui.spin(
                        ibox, self, "number_of_folds", 2, 100,
                        addToLayout=False,
                        callback=self.number_of_folds_changed))
        self.selected_fold_spin = gui.spin(
            ibox, self, "selectedFold", 1, self.number_of_folds,
            addToLayout=False, callback=self.fold_changed)
        form.addRow("Selected fold", self.selected_fold_spin)

        gui.appendRadioButton(sampling, "Boostrap")

        self.sql_box = gui.widgetBox(self.controlArea, "Sampling Type")
        sampling = gui.radioButtons(self.sql_box, self, "sampling_type",
                                    callback=self.sampling_type_changed)
        gui.appendRadioButton(sampling, "Time:")
        ibox = gui.indentedBox(sampling)
        spin = gui.spin(ibox, self, "sampleSizeSqlTime", minv=1, maxv=3600,
                        callback=set_sampling_type(self.SqlTime))
        spin.setSuffix(" sec")
        gui.appendRadioButton(sampling, "Percentage")
        ibox = gui.indentedBox(sampling)
        spin = gui.spin(ibox, self, "sampleSizeSqlPercentage", spinType=float,
                        minv=0.0001, maxv=100, step=0.1, decimals=4,
                        callback=set_sampling_type(self.SqlProportion))
        spin.setSuffix(" %")
        self.sql_box.setVisible(False)

        self.options_box = gui.widgetBox(self.controlArea, "Options")
        self.cb_seed = gui.checkBox(
            self.options_box, self, "use_seed",
            "Replicable (deterministic) sampling",
            callback=self.settings_changed)
        self.cb_stratify = gui.checkBox(
            self.options_box, self, "stratify",
            "Stratify sample (when possible)", callback=self.settings_changed)
        self.cb_sql_dl = gui.checkBox(
            self.options_box, self, "sql_dl", "Download data to local memory",
            callback=self.settings_changed)
        self.cb_sql_dl.setVisible(False)

        gui.button(self.controlArea, self, "Sample Data",
                   callback=self.commit, addSpace=8)
        self.controlArea.layout().addWidget(self.report_button)

    def sampling_type_changed(self):
        self.settings_changed()

    def number_of_folds_changed(self):
        self.selected_fold_spin.setMaximum(self.number_of_folds)
        self.sampling_type = self.CrossValidation
        self.settings_changed()

    def fold_changed(self):
        # a separate callback - if we decide to cache indices
        self.sampling_type = self.CrossValidation

    def settings_changed(self):
        self.indices = None

    def set_data(self, dataset):
        self.data = dataset
        if dataset is not None:
            sql = isinstance(dataset, SqlTable)
            self.sampling_box.setVisible(not sql)
            self.sql_box.setVisible(sql)
            self.cb_seed.setVisible(not sql)
            self.cb_stratify.setVisible(not sql)
            self.cb_sql_dl.setVisible(sql)
            self.dataInfoLabel.setText(
                '{}{} instances in input data set.'.format(*(
                    ('~', dataset.approx_len()) if sql else
                    ('', len(dataset)))))
            if not sql:
                self.sampleSizeSpin.setMaximum(len(dataset))
                self.updateindices()
        else:
            self.dataInfoLabel.setText('No data on input.')
            self.outputInfoLabel.setText('')
            self.indices = None
        self.commit()

    def commit(self):
        if self.data is None:
            sample = other = None
            self.sampled_instances = self.remaining_instances = None
            self.outputInfoLabel.setText("")
        elif isinstance(self.data, SqlTable):
            other = None
            if self.sampling_type == self.SqlProportion:
                sample = self.data.sample_percentage(
                    self.sampleSizeSqlPercentage, no_cache=True)
            else:
                sample = self.data.sample_time(
                    self.sampleSizeSqlTime, no_cache=True)
            if self.sql_dl:
                sample.download_data()
                sample = Table(sample)

        else:
            if self.indices is None or not self.use_seed:
                self.updateindices()
                if self.indices is None:
                    return
            if self.sampling_type in (
                    self.FixedProportion, self.FixedSize, self.Bootstrap):
                remaining, sample = self.indices
                self.outputInfoLabel.setText(
                    'Outputting %d instance%s.' %
                    (len(sample), "s" * (len(sample) != 1)))
            elif self.sampling_type == self.CrossValidation:
                remaining, sample = self.indices[self.selectedFold - 1]
                self.outputInfoLabel.setText(
                    'Outputting fold %d, %d instance%s.' %
                    (self.selectedFold, len(sample), "s" * (len(sample) != 1))
                )
            sample = self.data[sample]
            other = self.data[remaining]
            self.sampled_instances = len(sample)
            self.remaining_instances = len(other)
        self.send("Data Sample", sample)
        self.send("Remaining Data", other)

    def updateindices(self):
        err_msg = ""
        repl = True
        data_length = len(self.data)
        num_classes = len(self.data.domain.class_var.values) \
            if self.data.domain.has_discrete_class else 0

        size = None
        if self.sampling_type == self.FixedSize:
            size = self.sampleSizeNumber
            repl = self.replacement
        elif self.sampling_type == self.FixedProportion:
            size = np.ceil(self.sampleSizePercentage / 100 * data_length)
            repl = False
        elif self.sampling_type == self.CrossValidation:
            if data_length < self.number_of_folds:
                err_msg = "Number of folds exceeds the data size"
        else:
            assert self.sampling_type == self.Bootstrap

        if not repl and size is not None and (data_length <= size):
            err_msg = "Sample must be smaller than data"
        if not repl and data_length <= num_classes and self.stratify:
            err_msg = "Not enough data for stratified sampling"

        self.error(0)
        if err_msg:
            self.error(err_msg)
            self.indices = None
            return

        rnd = self.RandomSeed if self.use_seed else None
        stratified = (self.stratify and
                      type(self.data) == Table and
                      self.data.domain.has_discrete_class)
        if self.sampling_type == self.FixedSize:
            self.indices = sample_random_n(
                self.data, size,
                stratified=stratified, replace=self.replacement,
                random_state=rnd)
        elif self.sampling_type == self.FixedProportion:
            self.indices = sample_random_p(
                self.data, self.sampleSizePercentage / 100,
                stratified=stratified, random_state=rnd)
        elif self.sampling_type == self.Bootstrap:
            self.indices = sample_bootstrap(data_length, random_state=rnd)
        else:
            self.indices = sample_fold_indices(
                self.data, self.number_of_folds, stratified=stratified,
                random_state=rnd)

    def send_report(self):
        if self.sampling_type == self.FixedProportion:
            tpe = "Random sample with {} % of data".format(
                self.sampleSizePercentage)
        elif self.sampling_type == self.FixedSize:
            if self.sampleSizeNumber == 1:
                tpe = "Random data instance"
            else:
                tpe = "Random sample with {} data instances".format(
                    self.sampleSizeNumber)
                if self.replacement:
                    tpe += ", with replacement"
        elif self.sampling_type == self.CrossValidation:
            tpe = "Fold {} of {}-fold cross-validation".format(
                self.selectedFold, self.number_of_folds)
        else:
            tpe = "Undefined"  # should not come here at all
        if self.stratify:
            tpe += ", stratified (if possible)"
        if self.use_seed:
            tpe += ", deterministic"
        items = [("Sampling type", tpe)]
        if self.sampled_instances is not None:
            items += [
                ("Input", "{} instances".format(len(self.data))),
                ("Sample", "{} instances".format(self.sampled_instances)),
                ("Remaining", "{} instances".format(self.remaining_instances)),
            ]
        self.report_items(items)

def sample_fold_indices(table, folds=10, stratified=False, random_state=None):
    """
    :param Orange.data.Table table:
    :param int folds: Number of folds
    :param bool stratified: Return stratified indices (if applicable).
    :param Random random_state:
    :rval tuple-of-arrays: A tuple of array indices one for each fold.
    """
    if stratified and table.domain.has_discrete_class:
        # XXX: StratifiedKFold does not support random_state
        ind = skl_cross_validation.StratifiedKFold(
            table.Y.ravel(), folds, random_state=random_state)
    else:
        ind = skl_cross_validation.KFold(
            len(table), folds, shuffle=True, random_state=random_state)
    return tuple(ind)

def sample_random_n(table, n, stratified=False, replace=False,
                    random_state=None):
    if replace:
        if random_state is None:
            rgen = np.random
        else:
            rgen = np.random.mtrand.RandomState(random_state)
        sample = rgen.random_integers(0, len(table) - 1, n)
        o = np.ones(len(table))
        o[sample] = 0
        others = np.nonzero(o)[0]
        return others, sample
    if stratified and table.domain.has_discrete_class:
        test_size = max(len(table.domain.class_var.values), n)
        ind = skl_cross_validation.StratifiedShuffleSplit(
            table.Y.ravel(), n_iter=1,
            test_size=test_size, train_size=len(table) - test_size,
            random_state=random_state)
    else:
        ind = skl_cross_validation.ShuffleSplit(
            len(table), n_iter=1,
            test_size=n, random_state=random_state)
    return next(iter(ind))

def sample_random_p(table, p, stratified=False, random_state=None):
    n = int(math.ceil(len(table) * p))
    return sample_random_n(table, n, stratified, False, random_state)

def sample_bootstrap(size, random_state=None):
    rgen = np.random.RandomState(random_state)
    sample = rgen.randint(0, size, size)
    sample.sort()  # not needed for the code below, just for the user
    insample = np.ones((size,), dtype=np.bool)
    insample[sample] = False
    remaining = np.flatnonzero(insample)
    return remaining, sample

def test_main():
    app = QtGui.QApplication([])
    data = Table("iris")
    w = OWDataSampler()
    w.set_data(data)
    w.show()
    return app.exec_()

if __name__ == "__main__":
    sys.exit(test_main())


# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
#          Joel Nothman <joel.nothman@gmail.com>
#          Hamzeh Alsalhi <ha258@cornell.edu>
# License: BSD 3 clause

from collections import defaultdict
import itertools
import array

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator, TransformerMixin

from ..utils.fixes import np_version
from ..utils.fixes import sparse_min_max
from ..utils.fixes import astype
from ..utils.fixes import in1d
from ..utils import column_or_1d
from ..utils.validation import check_array
from ..utils.validation import check_is_fitted
from ..utils.validation import _num_samples
from ..utils.multiclass import unique_labels
from ..utils.multiclass import type_of_target

from ..externals import six

zip = six.moves.zip
map = six.moves.map

__all__ = [
    'label_binarize',
    'LabelBinarizer',
    'LabelEncoder',
    'MultiLabelBinarizer',
]

def _check_numpy_unicode_bug(labels):
    """Check that user is not subject to an old numpy bug

    Fixed in master before 1.7.0:

      https://github.com/numpy/numpy/pull/243

    """
    if np_version[:3] < (1, 7, 0) and labels.dtype.kind == 'U':
        raise RuntimeError("NumPy < 1.7.0 does not implement searchsorted"
                           " on unicode data correctly. Please upgrade"
                           " NumPy to use LabelEncoder with unicode inputs.")

class LabelEncoder(BaseEstimator, TransformerMixin):
    """Encode labels with value between 0 and n_classes-1.

    Read more in the :ref:`User Guide <preprocessing_targets>`.

    Attributes
    ----------
    classes_ : array of shape (n_class,)
        Holds the label for each class.

    Examples
    --------
    `LabelEncoder` can be used to normalize labels.

    >>> from sklearn import preprocessing
    >>> le = preprocessing.LabelEncoder()
    >>> le.fit([1, 2, 2, 6])
    LabelEncoder()
    >>> le.classes_
    array([1, 2, 6])
    >>> le.transform([1, 1, 2, 6]) #doctest: +ELLIPSIS
    array([0, 0, 1, 2]...)
    >>> le.inverse_transform([0, 0, 1, 2])
    array([1, 1, 2, 6])

    It can also be used to transform non-numerical labels (as long as they are
    hashable and comparable) to numerical labels.

    >>> le = preprocessing.LabelEncoder()
    >>> le.fit(["paris", "paris", "tokyo", "amsterdam"])
    LabelEncoder()
    >>> list(le.classes_)
    ['amsterdam', 'paris', 'tokyo']
    >>> le.transform(["tokyo", "tokyo", "paris"]) #doctest: +ELLIPSIS
    array([2, 2, 1]...)
    >>> list(le.inverse_transform([2, 2, 1]))
    ['tokyo', 'tokyo', 'paris']

    """

    def fit(self, y):
        """Fit label encoder

        Parameters
        ----------
        y : array-like of shape (n_samples,)
            Target values.

        Returns
        -------
        self : returns an instance of self.
        """
        y = column_or_1d(y, warn=True)
        _check_numpy_unicode_bug(y)
        self.classes_ = np.unique(y)
        return self

    def fit_transform(self, y):
        """Fit label encoder and return encoded labels

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        y = column_or_1d(y, warn=True)
        _check_numpy_unicode_bug(y)
        self.classes_, y = np.unique(y, return_inverse=True)
        return y

    def transform(self, y):
        """Transform labels to normalized encoding.

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')

        classes = np.unique(y)
        _check_numpy_unicode_bug(classes)
        if len(np.intersect1d(classes, self.classes_)) < len(classes):
            diff = np.setdiff1d(classes, self.classes_)
            raise ValueError("y contains new labels: %s" % str(diff))
        return np.searchsorted(self.classes_, y)

    def inverse_transform(self, y):
        """Transform labels back to original encoding.

        Parameters
        ----------
        y : numpy array of shape [n_samples]
            Target values.

        Returns
        -------
        y : numpy array of shape [n_samples]
        """
        check_is_fitted(self, 'classes_')

        diff = np.setdiff1d(y, np.arange(len(self.classes_)))
        if diff:
            raise ValueError("y contains new labels: %s" % str(diff))
        y = np.asarray(y)
        return self.classes_[y]

class LabelBinarizer(BaseEstimator, TransformerMixin):
    """Binarize labels in a one-vs-all fashion

    Several regression and binary classification algorithms are
    available in the scikit. A simple way to extend these algorithms
    to the multi-class classification case is to use the so-called
    one-vs-all scheme.

    At learning time, this simply consists in learning one regressor
    or binary classifier per class. In doing so, one needs to convert
    multi-class labels to binary labels (belong or does not belong
    to the class). LabelBinarizer makes this process easy with the
    transform method.

    At prediction time, one assigns the class for which the corresponding
    model gave the greatest confidence. LabelBinarizer makes this easy
    with the inverse_transform method.

    Read more in the :ref:`User Guide <preprocessing_targets>`.

    Parameters
    ----------

    neg_label : int (default: 0)
        Value with which negative labels must be encoded.

    pos_label : int (default: 1)
        Value with which positive labels must be encoded.

    sparse_output : boolean (default: False)
        True if the returned array from transform is desired to be in sparse
        CSR format.

    Attributes
    ----------

    classes_ : array of shape [n_class]
        Holds the label for each class.

    y_type_ : str,
        Represents the type of the target data as evaluated by
        utils.multiclass.type_of_target. Possible type are 'continuous',
        'continuous-multioutput', 'binary', 'multiclass',
        'mutliclass-multioutput', 'multilabel-indicator', and 'unknown'.

    sparse_input_ : boolean,
        True if the input data to transform is given as a sparse matrix, False
        otherwise.

    Examples
    --------
    >>> from sklearn import preprocessing
    >>> lb = preprocessing.LabelBinarizer()
    >>> lb.fit([1, 2, 6, 4, 2])
    LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)
    >>> lb.classes_
    array([1, 2, 4, 6])
    >>> lb.transform([1, 6])
    array([[1, 0, 0, 0],
           [0, 0, 0, 1]])

    Binary targets transform to a column vector

    >>> lb = preprocessing.LabelBinarizer()
    >>> lb.fit_transform(['yes', 'no', 'no', 'yes'])
    array([[1],
           [0],
           [0],
           [1]])

    Passing a 2D matrix for multilabel classification

    >>> import numpy as np
    >>> lb.fit(np.array([[0, 1, 1], [1, 0, 0]]))
    LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)
    >>> lb.classes_
    array([0, 1, 2])
    >>> lb.transform([0, 1, 2, 1])
    array([[1, 0, 0],
           [0, 1, 0],
           [0, 0, 1],
           [0, 1, 0]])

    See also
    --------
    label_binarize : function to perform the transform operation of
        LabelBinarizer with fixed classes.
    """

    def __init__(self, neg_label=0, pos_label=1, sparse_output=False):
        if neg_label >= pos_label:
            raise ValueError("neg_label={0} must be strictly less than "
                             "pos_label={1}.".format(neg_label, pos_label))

        if sparse_output and (pos_label == 0 or neg_label != 0):
            raise ValueError("Sparse binarization is only supported with non "
                             "zero pos_label and zero neg_label, got "
                             "pos_label={0} and neg_label={1}"
                             "".format(pos_label, neg_label))

        self.neg_label = neg_label
        self.pos_label = pos_label
        self.sparse_output = sparse_output

    def fit(self, y):
        """Fit label binarizer

        Parameters
        ----------
        y : numpy array of shape (n_samples,) or (n_samples, n_classes)
            Target values. The 2-d matrix should only contain 0 and 1,
            represents multilabel classification.

        Returns
        -------
        self : returns an instance of self.
        """
        self.y_type_ = type_of_target(y)
        if 'multioutput' in self.y_type_:
            raise ValueError("Multioutput target data is not supported with "
                             "label binarization")
        if _num_samples(y) == 0:
            raise ValueError('y has 0 samples: %r' % y)

        self.sparse_input_ = sp.issparse(y)
        self.classes_ = unique_labels(y)
        return self

    def transform(self, y):
        """Transform multi-class labels to binary labels

        The output of transform is sometimes referred to by some authors as the
        1-of-K coding scheme.

        Parameters
        ----------
        y : numpy array or sparse matrix of shape (n_samples,) or
            (n_samples, n_classes) Target values. The 2-d matrix should only
            contain 0 and 1, represents multilabel classification. Sparse
            matrix can be CSR, CSC, COO, DOK, or LIL.

        Returns
        -------
        Y : numpy array or CSR matrix of shape [n_samples, n_classes]
            Shape will be [n_samples, 1] for binary problems.
        """
        check_is_fitted(self, 'classes_')

        y_is_multilabel = type_of_target(y).startswith('multilabel')
        if y_is_multilabel and not self.y_type_.startswith('multilabel'):
            raise ValueError("The object was not fitted with multilabel"
                             " input.")

        return label_binarize(y, self.classes_,
                              pos_label=self.pos_label,
                              neg_label=self.neg_label,
                              sparse_output=self.sparse_output)

    def inverse_transform(self, Y, threshold=None):
        """Transform binary labels back to multi-class labels

        Parameters
        ----------
        Y : numpy array or sparse matrix with shape [n_samples, n_classes]
            Target values. All sparse matrices are converted to CSR before
            inverse transformation.

        threshold : float or None
            Threshold used in the binary and multi-label cases.

            Use 0 when:
                - Y contains the output of decision_function (classifier)
            Use 0.5 when:
                - Y contains the output of predict_proba

            If None, the threshold is assumed to be half way between
            neg_label and pos_label.

        Returns
        -------
        y : numpy array or CSR matrix of shape [n_samples] Target values.

        Notes
        -----
        In the case when the binary labels are fractional
        (probabilistic), inverse_transform chooses the class with the
        greatest value. Typically, this allows to use the output of a
        linear model's decision_function method directly as the input
        of inverse_transform.
        """
        check_is_fitted(self, 'classes_')

        if threshold is None:
            threshold = (self.pos_label + self.neg_label) / 2.

        if self.y_type_ == "multiclass":
            y_inv = _inverse_binarize_multiclass(Y, self.classes_)
        else:
            y_inv = _inverse_binarize_thresholding(Y, self.y_type_,
                                                   self.classes_, threshold)

        if self.sparse_input_:
            y_inv = sp.csr_matrix(y_inv)
        elif sp.issparse(y_inv):
            y_inv = y_inv.toarray()

        return y_inv

def label_binarize(y, classes, neg_label=0, pos_label=1, sparse_output=False):
    """Binarize labels in a one-vs-all fashion

    Several regression and binary classification algorithms are
    available in the scikit. A simple way to extend these algorithms
    to the multi-class classification case is to use the so-called
    one-vs-all scheme.

    This function makes it possible to compute this transformation for a
    fixed set of class labels known ahead of time.

    Parameters
    ----------
    y : array-like
        Sequence of integer labels or multilabel data to encode.

    classes : array-like of shape [n_classes]
        Uniquely holds the label for each class.

    neg_label : int (default: 0)
        Value with which negative labels must be encoded.

    pos_label : int (default: 1)
        Value with which positive labels must be encoded.

    sparse_output : boolean (default: False),
        Set to true if output binary array is desired in CSR sparse format

    Returns
    -------
    Y : numpy array or CSR matrix of shape [n_samples, n_classes]
        Shape will be [n_samples, 1] for binary problems.

    Examples
    --------
    >>> from sklearn.preprocessing import label_binarize
    >>> label_binarize([1, 6], classes=[1, 2, 4, 6])
    array([[1, 0, 0, 0],
           [0, 0, 0, 1]])

    The class ordering is preserved:

    >>> label_binarize([1, 6], classes=[1, 6, 4, 2])
    array([[1, 0, 0, 0],
           [0, 1, 0, 0]])

    Binary targets transform to a column vector

    >>> label_binarize(['yes', 'no', 'no', 'yes'], classes=['no', 'yes'])
    array([[1],
           [0],
           [0],
           [1]])

    See also
    --------
    LabelBinarizer : class used to wrap the functionality of label_binarize and
        allow for fitting to classes independently of the transform operation
    """
    if not isinstance(y, list):
        # XXX Workaround that will be removed when list of list format is
        # dropped
        y = check_array(y, accept_sparse='csr', ensure_2d=False, dtype=None)
    else:
        if _num_samples(y) == 0:
            raise ValueError('y has 0 samples: %r' % y)
    if neg_label >= pos_label:
        raise ValueError("neg_label={0} must be strictly less than "
                         "pos_label={1}.".format(neg_label, pos_label))

    if (sparse_output and (pos_label == 0 or neg_label != 0)):
        raise ValueError("Sparse binarization is only supported with non "
                         "zero pos_label and zero neg_label, got "
                         "pos_label={0} and neg_label={1}"
                         "".format(pos_label, neg_label))

    # To account for pos_label == 0 in the dense case
    pos_switch = pos_label == 0
    if pos_switch:
        pos_label = -neg_label

    y_type = type_of_target(y)
    if 'multioutput' in y_type:
        raise ValueError("Multioutput target data is not supported with label "
                         "binarization")
    if y_type == 'unknown':
        raise ValueError("The type of target data is not known")

    n_samples = y.shape[0] if sp.issparse(y) else len(y)
    n_classes = len(classes)
    classes = np.asarray(classes)

    if y_type == "binary":
        if len(classes) == 1:
            Y = np.zeros((len(y), 1), dtype=np.int)
            Y += neg_label
            return Y
        elif len(classes) >= 3:
            y_type = "multiclass"

    sorted_class = np.sort(classes)
    if (y_type == "multilabel-indicator" and classes.size != y.shape[1]):
        raise ValueError("classes {0} missmatch with the labels {1}"
                         "found in the data".format(classes, unique_labels(y)))

    if y_type in ("binary", "multiclass"):
        y = column_or_1d(y)

        # pick out the known labels from y
        y_in_classes = in1d(y, classes)
        y_seen = y[y_in_classes]
        indices = np.searchsorted(sorted_class, y_seen)
        indptr = np.hstack((0, np.cumsum(y_in_classes)))

        data = np.empty_like(indices)
        data.fill(pos_label)
        Y = sp.csr_matrix((data, indices, indptr),
                          shape=(n_samples, n_classes))
    elif y_type == "multilabel-indicator":
        Y = sp.csr_matrix(y)
        if pos_label != 1:
            data = np.empty_like(Y.data)
            data.fill(pos_label)
            Y.data = data
    else:
        raise ValueError("%s target data is not supported with label "
                         "binarization" % y_type)

    if not sparse_output:
        Y = Y.toarray()
        Y = astype(Y, int, copy=False)

        if neg_label != 0:
            Y[Y == 0] = neg_label

        if pos_switch:
            Y[Y == pos_label] = 0
    else:
        Y.data = astype(Y.data, int, copy=False)

    # preserve label ordering
    if np.any(classes != sorted_class):
        indices = np.searchsorted(sorted_class, classes)
        Y = Y[:, indices]

    if y_type == "binary":
        if sparse_output:
            Y = Y.getcol(-1)
        else:
            Y = Y[:, -1].reshape((-1, 1))

    return Y

def _inverse_binarize_multiclass(y, classes):
    """Inverse label binarization transformation for multiclass.

    Multiclass uses the maximal score instead of a threshold.
    """
    classes = np.asarray(classes)

    if sp.issparse(y):
        # Find the argmax for each row in y where y is a CSR matrix

        y = y.tocsr()
        n_samples, n_outputs = y.shape
        outputs = np.arange(n_outputs)
        row_max = sparse_min_max(y, 1)[1]
        row_nnz = np.diff(y.indptr)

        y_data_repeated_max = np.repeat(row_max, row_nnz)
        # picks out all indices obtaining the maximum per row
        y_i_all_argmax = np.flatnonzero(y_data_repeated_max == y.data)

        # For corner case where last row has a max of 0
        if row_max[-1] == 0:
            y_i_all_argmax = np.append(y_i_all_argmax, [len(y.data)])

        # Gets the index of the first argmax in each row from y_i_all_argmax
        index_first_argmax = np.searchsorted(y_i_all_argmax, y.indptr[:-1])
        # first argmax of each row
        y_ind_ext = np.append(y.indices, [0])
        y_i_argmax = y_ind_ext[y_i_all_argmax[index_first_argmax]]
        # Handle rows of all 0
        y_i_argmax[np.where(row_nnz == 0)[0]] = 0

        # Handles rows with max of 0 that contain negative numbers
        samples = np.arange(n_samples)[(row_nnz > 0) &
                                       (row_max.ravel() == 0)]
        for i in samples:
            ind = y.indices[y.indptr[i]:y.indptr[i + 1]]
            y_i_argmax[i] = classes[np.setdiff1d(outputs, ind)][0]

        return classes[y_i_argmax]
    else:
        return classes.take(y.argmax(axis=1), mode="clip")

def _inverse_binarize_thresholding(y, output_type, classes, threshold):
    """Inverse label binarization transformation using thresholding."""

    if output_type == "binary" and y.ndim == 2 and y.shape[1] > 2:
        raise ValueError("output_type='binary', but y.shape = {0}".
                         format(y.shape))

    if output_type != "binary" and y.shape[1] != len(classes):
        raise ValueError("The number of class is not equal to the number of "
                         "dimension of y.")

    classes = np.asarray(classes)

    # Perform thresholding
    if sp.issparse(y):
        if threshold > 0:
            if y.format not in ('csr', 'csc'):
                y = y.tocsr()
            y.data = np.array(y.data > threshold, dtype=np.int)
            y.eliminate_zeros()
        else:
            y = np.array(y.toarray() > threshold, dtype=np.int)
    else:
        y = np.array(y > threshold, dtype=np.int)

    # Inverse transform data
    if output_type == "binary":
        if sp.issparse(y):
            y = y.toarray()
        if y.ndim == 2 and y.shape[1] == 2:
            return classes[y[:, 1]]
        else:
            if len(classes) == 1:
                y = np.empty(len(y), dtype=classes.dtype)
                y.fill(classes[0])
                return y
            else:
                return classes[y.ravel()]

    elif output_type == "multilabel-indicator":
        return y

    else:
        raise ValueError("{0} format is not supported".format(output_type))

class MultiLabelBinarizer(BaseEstimator, TransformerMixin):
    """Transform between iterable of iterables and a multilabel format

    Although a list of sets or tuples is a very intuitive format for multilabel
    data, it is unwieldy to process. This transformer converts between this
    intuitive format and the supported multilabel format: a (samples x classes)
    binary matrix indicating the presence of a class label.

    Parameters
    ----------
    classes : array-like of shape [n_classes] (optional)
        Indicates an ordering for the class labels

    sparse_output : boolean (default: False),
        Set to true if output binary array is desired in CSR sparse format

    Attributes
    ----------
    classes_ : array of labels
        A copy of the `classes` parameter where provided,
        or otherwise, the sorted set of classes found when fitting.

    Examples
    --------
    >>> mlb = MultiLabelBinarizer()
    >>> mlb.fit_transform([(1, 2), (3,)])
    array([[1, 1, 0],
           [0, 0, 1]])
    >>> mlb.classes_
    array([1, 2, 3])

    >>> mlb.fit_transform([set(['sci-fi', 'thriller']), set(['comedy'])])
    array([[0, 1, 1],
           [1, 0, 0]])
    >>> list(mlb.classes_)
    ['comedy', 'sci-fi', 'thriller']

    """
    def __init__(self, classes=None, sparse_output=False):
        self.classes = classes
        self.sparse_output = sparse_output

    def fit(self, y):
        """Fit the label sets binarizer, storing `classes_`

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        self : returns this MultiLabelBinarizer instance
        """
        if self.classes is None:
            classes = sorted(set(itertools.chain.from_iterable(y)))
        else:
            classes = self.classes
        dtype = np.int if all(isinstance(c, int) for c in classes) else object
        self.classes_ = np.empty(len(classes), dtype=dtype)
        self.classes_[:] = classes
        return self

    def fit_transform(self, y):
        """Fit the label sets binarizer and transform the given label sets

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        y_indicator : array or CSR matrix, shape (n_samples, n_classes)
            A matrix such that `y_indicator[i, j] = 1` iff `classes_[j]` is in
            `y[i]`, and 0 otherwise.
        """
        if self.classes is not None:
            return self.fit(y).transform(y)

        # Automatically increment on new class
        class_mapping = defaultdict(int)
        class_mapping.default_factory = class_mapping.__len__
        yt = self._transform(y, class_mapping)

        # sort classes and reorder columns
        tmp = sorted(class_mapping, key=class_mapping.get)

        # (make safe for tuples)
        dtype = np.int if all(isinstance(c, int) for c in tmp) else object
        class_mapping = np.empty(len(tmp), dtype=dtype)
        class_mapping[:] = tmp
        self.classes_, inverse = np.unique(class_mapping, return_inverse=True)
        yt.indices = np.take(inverse, yt.indices)

        if not self.sparse_output:
            yt = yt.toarray()

        return yt

    def transform(self, y):
        """Transform the given label sets

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        y_indicator : array or CSR matrix, shape (n_samples, n_classes)
            A matrix such that `y_indicator[i, j] = 1` iff `classes_[j]` is in
            `y[i]`, and 0 otherwise.
        """
        class_to_index = dict(zip(self.classes_, range(len(self.classes_))))
        yt = self._transform(y, class_to_index)

        if not self.sparse_output:
            yt = yt.toarray()

        return yt

    def _transform(self, y, class_mapping):
        """Transforms the label sets with a given mapping

        Parameters
        ----------
        y : iterable of iterables
        class_mapping : Mapping
            Maps from label to column index in label indicator matrix

        Returns
        -------
        y_indicator : sparse CSR matrix, shape (n_samples, n_classes)
            Label indicator matrix
        """
        indices = array.array('i')
        indptr = array.array('i', [0])
        for labels in y:
            indices.extend(set(class_mapping[label] for label in labels))
            indptr.append(len(indices))
        data = np.ones(len(indices), dtype=int)

        return sp.csr_matrix((data, indices, indptr),
                             shape=(len(indptr) - 1, len(class_mapping)))

    def inverse_transform(self, yt):
        """Transform the given indicator matrix into label sets

        Parameters
        ----------
        yt : array or sparse matrix of shape (n_samples, n_classes)
            A matrix containing only 1s ands 0s.

        Returns
        -------
        y : list of tuples
            The set of labels for each sample such that `y[i]` consists of
            `classes_[j]` for each `yt[i, j] == 1`.
        """
        if yt.shape[1] != len(self.classes_):
            raise ValueError('Expected indicator for {0} classes, but got {1}'
                             .format(len(self.classes_), yt.shape[1]))

        if sp.issparse(yt):
            yt = yt.tocsr()
            if len(yt.data) != 0 and len(np.setdiff1d(yt.data, [0, 1])) > 0:
                raise ValueError('Expected only 0s and 1s in label indicator.')
            return [tuple(self.classes_.take(yt.indices[start:end]))
                    for start, end in zip(yt.indptr[:-1], yt.indptr[1:])]
        else:
            unexpected = np.setdiff1d(yt, [0, 1])
            if len(unexpected) > 0:
                raise ValueError('Expected only 0s and 1s in label indicator. '
                                 'Also got {0}'.format(unexpected))
            return [tuple(self.classes_.compress(indicators)) for indicators
                    in yt]


"""
The :mod:`sklearn.metrics.cluster` submodule contains evaluation metrics for
cluster analysis results. There are two forms of evaluation:

- supervised, which uses a ground truth class values for each sample.
- unsupervised, which does not and measures the 'quality' of the model itself.
"""
from .supervised import adjusted_mutual_info_score
from .supervised import normalized_mutual_info_score
from .supervised import adjusted_rand_score
from .supervised import completeness_score
from .supervised import contingency_matrix
from .supervised import expected_mutual_information
from .supervised import homogeneity_completeness_v_measure
from .supervised import homogeneity_score
from .supervised import mutual_info_score
from .supervised import v_measure_score
from .supervised import entropy
from .unsupervised import silhouette_samples
from .unsupervised import silhouette_score
from .bicluster import consensus_score

__all__ = ["adjusted_mutual_info_score", "normalized_mutual_info_score",
           "adjusted_rand_score", "completeness_score", "contingency_matrix",
           "expected_mutual_information", "homogeneity_completeness_v_measure",
           "homogeneity_score", "mutual_info_score", "v_measure_score",
           "entropy", "silhouette_samples", "silhouette_score",
           "consensus_score"]


"""Parallel utility function using joblib

copied from https://github.com/mne-tools/mne-python

Author: Alexandre Gramfort <gramfort@nmr.mgh.harvard.edu>
License: Simplified BSD

changes for statsmodels (Josef Perktold)
- try import from joblib directly, (doesn't import all of sklearn)

"""
from __future__ import print_function

from statsmodels.tools.sm_exceptions import (ModuleUnavailableWarning,
                                             module_unavailable_doc)

def parallel_func(func, n_jobs, verbose=5):
    """Return parallel instance with delayed function

    Util function to use joblib only if available

    Parameters
    ----------
    func: callable
        A function
    n_jobs: int
        Number of jobs to run in parallel
    verbose: int
        Verbosity level

    Returns
    -------
    parallel: instance of joblib.Parallel or list
        The parallel object
    my_func: callable
        func if not parallel or delayed(func)
    n_jobs: int
        Number of jobs >= 0

    Examples
    --------
    >>> from math import sqrt
    >>> from statsmodels.tools.parallel import parallel_func
    >>> parallel, p_func, n_jobs = parallel_func(sqrt, n_jobs=-1, verbose=0)
    >>> print(n_jobs)
    >>> parallel(p_func(i**2) for i in range(10))
    """
    try:
        try:
            from joblib import Parallel, delayed
        except ImportError:
            from sklearn.externals.joblib import Parallel, delayed

        parallel = Parallel(n_jobs, verbose=verbose)
        my_func = delayed(func)

        if n_jobs == -1:
            try:
                import multiprocessing
                n_jobs = multiprocessing.cpu_count()
            except (ImportError, NotImplementedError):
                import warnings
                warnings.warn(module_unavailable_doc.format('multiprocessing'),
                              ModuleUnavailableWarning)
                n_jobs = 1

    except ImportError:
        import warnings
        warnings.warn(module_unavailable_doc.format('joblib'),
                      ModuleUnavailableWarning)
        n_jobs = 1
        my_func = func
        parallel = list
    return parallel, my_func, n_jobs


import unittest
import os
import shutil
import json
import tempfile
import uuid

from mleap.sklearn.tree.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.datasets import load_iris

class TransformerTests(unittest.TestCase):
    def setUp(self):
        self.tmp_dir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmp_dir)

    def test_decision_tree_classifier(self):
        X = [[0, 1], [1, 1]]
        Y = [0, 0]

        dt_classifier = DecisionTreeClassifier()
        dt_classifier = dt_classifier.fit(X, Y)
        dt_classifier.mlinit(input_features ='feature', prediction_column = 'pred', feature_names = ['a'])
        dt_classifier.serialize_to_bundle(self.tmp_dir, dt_classifier.name)

        expected_model = {
            "attributes": {
                "num_features": {
                    "long": 2
                },
                "num_classes": {
                    "long": 1
                }
            },
            "op": "decision_tree_classifier"
        }

        # Test model.json
        with open("{}/{}.node/model.json".format(self.tmp_dir, dt_classifier.name)) as json_data:
            model = json.load(json_data)

        self.assertEqual(dt_classifier.op, expected_model['op'])
        self.assertEqual(expected_model['attributes']['num_features']['long'], model['attributes']['num_features']['long'])
        self.assertEqual(expected_model['attributes']['num_classes']['long'], model['attributes']['num_classes']['long'])

        # Test node.json
        with open("{}/{}.node/node.json".format(self.tmp_dir, dt_classifier.name)) as json_data:
            node = json.load(json_data)

        self.assertEqual(dt_classifier.name, node['name'])
        self.assertEqual(dt_classifier.input_features, node['shape']['inputs'][0]['name'])
        self.assertEqual(dt_classifier.prediction_column, node['shape']['outputs'][0]['name'])

    def test_decision_tree_classifier_with_iris_dataset(self):
        iris = load_iris()

        dt_classifier = DecisionTreeClassifier()
        dt_classifier.mlinit(input_features = 'features', prediction_column = 'species', feature_names = iris.feature_names)
        dt_classifier = dt_classifier.fit(iris.data, iris.target)
        dt_classifier.serialize_to_bundle(self.tmp_dir, dt_classifier.name)

        expected_model = {
            "attributes": {
                "num_features": {
                    "long": 4
                },
                "num_classes": {
                    "long": 3
                }
            },
            "op": "decision_tree_classifier"
        }

        # Test model.json
        with open("{}/{}.node/model.json".format(self.tmp_dir, dt_classifier.name)) as json_data:
            model = json.load(json_data)

        self.assertEqual(dt_classifier.op, expected_model['op'])
        self.assertEqual(expected_model['attributes']['num_features']['long'], model['attributes']['num_features']['long'])
        self.assertEqual(expected_model['attributes']['num_classes']['long'], model['attributes']['num_classes']['long'])

        # Test node.json
        with open("{}/{}.node/node.json".format(self.tmp_dir, dt_classifier.name)) as json_data:
            node = json.load(json_data)

        self.assertEqual(dt_classifier.name, node['name'])
        self.assertEqual(dt_classifier.input_features, node['shape']['inputs'][0]['name'])
        self.assertEqual(dt_classifier.prediction_column, node['shape']['outputs'][0]['name'])


import csv
from sklearn import tree
import editdist
import re

def string_match_score(p1,p2,field):
    s1 = p1[field]
    s2 = p2[field]
    return editdist.distance(s1.lower(),s2.lower())/float(len(s1))

def jaccard_score(p1,p2,field):
    name1 = p1[field]
    name2 = p2[field]
    set1 = set(name1.lower().split())
    set2 = set(name2.lower().split())
    c = set1.intersection(set2)
    return float(len(c)) / (len(set1) + len(set2) - len(c))

def price_score(p1,p2,field):
    price1 = p1[field]
    if (len(price1) == 0): return 10000
    price2 = p2[field]
    if (len(price2) == 0): return 10000
    price1 = re.sub('[\$,]', '', price1)
    price2 = re.sub('[\$,]', '', price2)
    price1 = float(price1)
    price2 = float(price2)
    return abs(price1 - price2)

print "Loading Data"

abtReader = csv.DictReader(open("Abt.csv","rU"))
buyReader = csv.DictReader(open("Buy.csv","rU"))
gtLines = csv.DictReader(open("abt_buy_perfectMapping.csv","rU"))
gtBuyMap = {}
gtAbtMap = {}
abtAr = []
buyAr = []

for r in abtReader:
    abtAr.append(r)

for r in buyReader:
    buyAr.append(r)

for r in gtLines:
    gtAbtMap[r["idAbt"]] = r["idBuy"]
    gtBuyMap[r["idBuy"]] = r["idAbt"]

for loop in range(0,10,1):
    falsePos = 0
    truePos = 0
    falseNeg = 0
    trueNeg = 0
    thresh = float(loop)/10.0
    for r1 in buyAr:
        bestMatch = 0
        bestVal = []
        j = 0
        for r2 in abtAr:
            s = jaccard_score(r1,r2,"name")
            if (s > bestMatch):
                bestMatch = s
                bestVal = r2
        if (bestMatch > thresh):
            #        print "Best match: ",r1["name"],bestVal["name"],"score=",bestMatch
            if (gtBuyMap[r1["id"]] == bestVal["id"]):
                truePos = truePos + 1
            else:
                falsePos = falsePos + 1

    precision = truePos / float(truePos + falsePos)
    recall = truePos / float(len(buyAr))
    fmeas = (2.0 * precision * recall) / (precision + recall)

    print "THRESH = ",thresh,"TP = ",truePos,"FP = ",falsePos,"PREC = ",precision,"RECALL = ",recall,"F = ",fmeas


# -*- coding: utf-8 -*-
__title__ = 'latinpigsay'
__license__ = 'MIT'
__author__ = 'Steven Cutting'
__author_email__ = 'steven.c.projects@gmail.com'
__created_on__ = '12/3/2014'

"""
Created on Wed Dec  3 17:36:17 2014

@author: steven_c
"""

acidtest = """Can you talk piglatin to piglatin.
"""

quotes = """A Tale of Two Cities LITE(tm)
     -- by Charles Dickens

     A lawyer who looks like a French Nobleman is executed in his place.

The Metamorphosis LITE(tm)
     -- by Franz Kafka

     A man turns into a bug and his family gets annoyed.

Lord of the Rings LITE(tm)
     -- by J. R. R. Tolkien

     Some guys take a long vacation to throw a ring into a volcano.

Hamlet LITE(tm)
     -- by Wm. Shakespeare

     A college student on vacation with family problems, a screwy
     girl-friend and a mother who won't act her age.
"""

paragraphs = """For many people (myself among them), the Python language is easy to fall in love with.
Since its first appearance in 1991, Python has become one of the most popular dynamic,
programming languages, along with Perl, Ruby, and others. Python and Ruby have
become especially popular in recent years for building websites using their numerous
web frameworks, like Rails (Ruby) and Django (Python). Such languages are often
called scripting languages as they can be used to write quick-and-dirty small programs,
or scripts. I don’t like the term “scripting language” as it carries a connotation that they
cannot be used for building mission-critical software. Among interpreted languages
Python is distinguished by its large and active scientific computing community.
Adoption of Python for scientific computing in both industry applications and academic
research has increased significantly since the early 2000s.

For data analysis and interactive, exploratory computing and data visualization, Python
will inevitably draw comparisons with the many other domain-specific open source
and commercial programming languages and tools in wide use, such as R, MATLAB,
SAS, Stata, and others. In recent years, Python’s improved library support (primarily
pandas) has made it a strong alternative for data manipulation tasks. Combined with
Python’s strength in general purpose programming, it is an excellent choice as a single
language for building data-centric applications.
"""

simplepgs = """Simple test.
Paragraphs. test.
Derp, derp   a.

Simple test.
Let's sentence ma'am let's full of ain't contractions I'm i'm couldn't've I'd.
Fred's stapler.
Fred's going to the movie.
O'clock o'clock.
Paragraphs. test.
Derp, derp.
"""

contsentence = "Let's sentence ma'am let's full of ain't contractions I'm i'm couldn't've I'd."

sentence = 'If capture groups are used, then the matched text is also included in the result.'

listofwords = ['Pig Latin',
               'hello',
               'switch',
               'glove',
               'fruit smoothie',
               'egg',
               'ultimate',
               'I',
               'yellow',
               'my',
               'rhythm',
               '436',
               '5',
               ]

txt = """
The Gettysburg Address

Four score and seven years ago our fathers brought forth on this continent,
a new nation, conceived in Liberty, and dedicated to the proposition that all
men are created equal.

Now we are engaged in a great civil war, testing whether that nation, or any
nation so conceived and so dedicated, can long endure. We are met on a great
battlefield of that war. We have come to dedicate a portion of that field, as
a final resting place for those who here gave their lives that that nation
might live. It is altogether fitting and proper that we should do this.

But, in a larger sense, we cannot dedicate - we cannot consecrate - we cannot
hallow - this ground. The brave men, living and dead, who struggled here, have
consecrated it, far above our poor power to add or detract. The world will
little note, nor long remember what we say here, but it can never forget what
they did here. It is for us the living, rather, to be dedicated here to the
unfinished work which they who fought here have thus far so nobly advanced.
It is rather for us to be here dedicated to the great task remaining before
us - that from these honored dead we take increased devotion to that cause for
which they gave the last full measure of devotion - that we here highly resolve
that these dead shall not have died in vain - that this nation, under God,
shall have a new birth of freedom - and that government of the people, by
the people, for the people, shall not perish from the earth.
"""

paragraphs_og = """For many people (myself among them), the Python language is easy to fall in love with.
Since its first appearance in 1991, Python has become one of the most popular dynamic,
programming languages, along with Perl, Ruby, and others. Python and Ruby have
become especially popular in recent years for building websites using their numerous
web frameworks, like Rails (Ruby) and Django (Python). Such languages are often
called scripting languages as they can be used to write quick-and-dirty small programs,
or scripts. I don’t like the term “scripting language” as it carries a connotation that they
cannot be used for building mission-critical software. Among interpreted languages
Python is distinguished by its large and active scientific computing community. Adop-
tion of Python for scientific computing in both industry applications and academic
research has increased significantly since the early 2000s.
For data analysis and interactive, exploratory computing and data visualization, Python
will inevitably draw comparisons with the many other domain-specific open source
and commercial programming languages and tools in wide use, such as R, MATLAB,
SAS, Stata, and others. In recent years, Python’s improved library support (primarily
pandas) has made it a strong alternative for data manipulation tasks. Combined with
Python’s strength in general purpose programming, it is an excellent choice as a single
language for building data-centric applications.
"""


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 22 22:52:24 2015

@author: lorraine
"""

import json
from pprint import pprint
import numpy as np
from scipy.stats import mstats
from scipy import stats
import csv
import pandas as pd
#json_data=open("data/{0}_B.json".format("pizza")).read()

#data = json.loads(json_data)
#pprint(data)

def normaltest_data(category):
    data,population = load_rating_data(category)
    z,pval = mstats.normaltest(data)
    print(category+" p value is "+str(pval))
    if(pval < 0.01):
        print "Not normal distribution"
    else:
        print "normal"

# normaltest_data
# null hypothesis is the pizza ratings all over the states follow a normal distribution
# A a significan level of 0.01 was chosen.
#Since the calculated p value is greater than the significan level, we do not reject the null hypothesis
#Therefore we can safely assume the ratings follows a normal distribution

#Suppose the top-40 rated pizza rating nationwide is 4.0, the one sample t-test returns a p value of 0.0019 < significance level=0.05
#therefore we can reject the null hypothesis. Do not have sufficient evidence to conclude the population mean is 4.0
#one-sided t-test, H0: score = 4.0, H1: score < 4.0
# As t<0 & p/2<alpha, we reject null hypothesis. Enough evidence to conclude best pizza score < 4.0

#assume the best pizza and best chinese have the same score in the population
#p-val = 2.32e-07 < 0.01, reject the null hypothesis. Do not have sufficient confidence to conclude the best scores are the same
#One-tailed greater than test. H0: pizza = chinese, H1:pizza >= chinese.
#As t>0 p/2<alpha, we reject null hypothesis. Enough evidence to conclude that best pizza socre is significantly greater than best chinese food

#two side p-val=0.003<0.01, t>0, reject null
#H0: best pizza score = best mexican, H1:best pizza >= mexican
#As t>0 and p/2<alpha, we reject null hypothesis. Best pizza is significantly greater than best mexican

#H0: best chinese = best mexican
#H1: best chinese not equal
# p>0.01, do not reject null. Mexican rating is not significantly different than Chinese

#assume the best pizza and the best bar have the same score in the population
#p-val=0.64 > 0.05, do ont reject the null hyphothesis. The best bar score is not significantly different from best pizza
def anova_test(cat1,cat2,cat3,cat4):
    x1,pop1=load_rating_data(cat1)
    x2,pop2=load_rating_data(cat2)
    x3,pop3=load_rating_data(cat3)
    x4,pop4=load_rating_data(cat4)
    F_val, p_val_anova = stats.f_oneway(x1,x2,x3,x4)
    print("anova f val"+str(F_val))
    print("anova p val"+str(p_val_anova))
# anova test null hypothesis:the population mean of the best pizza, bar, chinese and mexican restaurant ratings are the same
#p_val=1.13e-05<0.01, reject null hypothesis
#need to state the assumption of Anova Test
def pearson_rapop(category):
    rating,population = load_rating_data(category)
    pearson, p_val = stats.pearsonr(rating,population)
    print("pearson rapop is "+str(pearson))
    print("pearson rapop p_val is "+str(p_val))
# pearson coefficient = 0.23, 0.20<pearson<0.29,weak positive correlation
# p_val=0.09>0.05, H0: There is so statistically significant relationship between the two variables
# do not reject null hypothesis

def load_rating_data(category):
    with open("data/{0}_B.json".format(category),"r") as f:
        cat = f.read()
    cat = json.loads(cat)
    rating=[]
    population=[]
    for i in xrange(len(cat[category])):
        score = cat[category][i].values()
        rating.append(score[0]["rating"])
        population.append(score[0]["population"])

    return rating,population

def pearson_raAge(category):
    rating,population = load_rating_data(category)
    rating = np.array(rating)
    population=np.array(population)
    age = []
    f = open('data/MedianAge.csv')
    csv_f = csv.reader(f)
    for row in csv_f:
        age.append(float(row[2]))
    #rating = np.array(rating)
    age=np.array(age)
    pearson, p_val = stats.pearsonr(rating,age)
    print("pearson raAge is "+str(pearson))
    print("pearson raAge p_val is "+str(p_val))
#neglible correlation between rating and median age

def one_sample_ttest(category,base):
    rating,population=load_rating_data(category)
    rating = np.array(rating)
    population=np.array(population)
    t4, prob4 = stats.ttest_1samp(rating,base)
    print("t value of "+category+str(t4))
    print("p value of "+category+str(prob4))

def two_sample_ttest(category1, category2):
    data1,populaton1=load_rating_data(category1)
    data1 = np.array(data1)
    data2,population2=load_rating_data(category2)
    data2 = np.array(data2)
    t, prob = stats.ttest_rel(data1,data2)
    print("t value of "+ category1+category2+str(t))
    print("p value of "+ category1+category2+str(prob))

category_filter = ["pizza","chinese","mexican","bars"]

#for category　in category_filter:
normaltest_data("pizza")
# pearson_raAge("pizza")
# pearson_rapop("pizza")
# one_sample_ttest("pizza",4)
# two_sample_ttest("pizza","chinese")
# anova_test("pizza","chinese","mexican","bars")


"""
=================================================
Concatenating multiple feature extraction methods
=================================================

In many real-world examples, there are many ways to extract features from a
dataset. Often it is beneficial to combine several methods to obtain good
performance. This example shows how to use ``FeatureUnion`` to combine
features obtained by PCA and univariate selection.

Combining features using this transformer has the benefit that it allows
cross validation and grid searches over the whole process.

The combination used in this example is not particularly helpful on this
dataset and is only used to illustrate the usage of FeatureUnion.
"""

# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#
# License: BSD 3 clause

from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.grid_search import GridSearchCV
from sklearn.svm import SVC
from sklearn.datasets import load_iris
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest

iris = load_iris()

X, y = iris.data, iris.target

# This dataset is way to high-dimensional. Better do PCA:
pca = PCA(n_components=2)

# Maybe some original features where good, too?
selection = SelectKBest(k=1)

# Build estimator from PCA and Univariate selection:

combined_features = FeatureUnion([("pca", pca), ("univ_select", selection)])

# Use combined features to transform dataset:
X_features = combined_features.fit(X, y).transform(X)

svm = SVC(kernel="linear")

# Do grid search over k, n_components and C:

pipeline = Pipeline([("features", combined_features), ("svm", svm)])

param_grid = dict(features__pca__n_components=[1, 2, 3],
                  features__univ_select__k=[1, 2],
                  svm__C=[0.1, 1, 10])

grid_search = GridSearchCV(pipeline, param_grid=param_grid, verbose=10)
grid_search.fit(X, y)
print(grid_search.best_estimator_)


import pandas as pd

from bokeh.io import output_file, show
from bokeh.models import BasicTicker, ColorBar, ColumnDataSource, LinearColorMapper, PrintfTickFormatter
from bokeh.plotting import figure
from bokeh.sampledata.unemployment1948 import data
from bokeh.transform import transform

output_file("unemploymemt.html")

data.Year = data.Year.astype(str)
data = data.set_index('Year')
data.drop('Annual', axis=1, inplace=True)
data.columns.name = 'Month'

# reshape to 1D array or rates with a month and year for each row.
df = pd.DataFrame(data.stack(), columns=['rate']).reset_index()

source = ColumnDataSource(df)

# this is the colormap from the original NYTimes plot
colors = ["#75968f", "#a5bab7", "#c9d9d3", "#e2e2e2", "#dfccce", "#ddb7b1", "#cc7878", "#933b41", "#550b1d"]
mapper = LinearColorMapper(palette=colors, low=df.rate.min(), high=df.rate.max())

p = figure(plot_width=800, plot_height=300, title="US Unemployment 1948—2016",
           x_range=list(data.index), y_range=list(reversed(data.columns)),
           toolbar_location=None, tools="", x_axis_location="above")

p.rect(x="Year", y="Month", width=1, height=1, source=source,
       line_color=None, fill_color=transform('rate', mapper))

color_bar = ColorBar(color_mapper=mapper, location=(0, 0),
                     ticker=BasicTicker(desired_num_ticks=len(colors)),
                     formatter=PrintfTickFormatter(format="%d%%"))

p.add_layout(color_bar, 'right')

p.axis.axis_line_color = None
p.axis.major_tick_line_color = None
p.axis.major_label_text_font_size = "5pt"
p.axis.major_label_standoff = 0
p.xaxis.major_label_orientation = 1.0

show(p)


from __future__ import print_function, division
import numpy as np
import matplotlib.pyplot as plt
import vcfnp
vcfnp.__version__

filename = 'fixture/sample.vcf'

# load data from fixed fields (including INFO)
v = vcfnp.variants(filename, cache=True).view(np.recarray)

# print some simple variant metrics
print('found %s variants (%s SNPs)' % (v.size, np.count_nonzero(v.is_snp)))
print('QUAL mean (std): %s (%s)' % (np.mean(v.QUAL), np.std(v.QUAL)))

# plot a histogram of variant depth
fig = plt.figure(1)
ax = fig.add_subplot(111)
ax.hist(v.DP)
ax.set_title('DP histogram')
ax.set_xlabel('DP')
plt.show()

# load data from sample columns
c = vcfnp.calldata_2d(filename, cache=True).view(np.recarray)

# print some simple genotype metrics
count_phased = np.count_nonzero(c.is_phased)
count_variant = np.count_nonzero(np.any(c.genotype > 0, axis=2))
count_missing = np.count_nonzero(~c.is_called)
print('calls (phased, variant, missing): %s (%s, %s, %s)'
    % (c.flatten().size, count_phased, count_variant, count_missing))

# plot a histogram of genotype quality
fig = plt.figure(2)
ax = fig.add_subplot(111)
ax.hist(c.GQ.flatten())
ax.set_title('GQ histogram')
ax.set_xlabel('GQ')
plt.show()


from math import fabs, isnan
from datetime import datetime
from spatiotemporal.unix_time import UnixTime
from utility.generic import convert_dict_to_sorted_lists
from utility.numeric.globals import EPSILON
from numpy import NINF as NEGATIVE_INFINITY, PINF as POSITIVE_INFINITY
from scipy.integrate import quad

__author__ = 'keyvan'

def integral(function, start, end):
    if hasattr(function, 'integral'):
        return function.integral(start, end)
    area, error = quad(function, start, end)
    return area

def almost_equals(a, b, epsilon=EPSILON):
    if fabs(a - b) < epsilon:
        return True
    return False

def invoke_method_on(method, sequence_or_point):
    if method is None:
        return None
    if not callable(method):
        raise TypeError("'method' is not callable")
    result = []
    try:
        for point in sequence_or_point:
            if type(point) is datetime:
                point = UnixTime(point)
            result.append(method(point))
    except TypeError:
        if type(sequence_or_point) is datetime:
            sequence_or_point = UnixTime(sequence_or_point)
        return method(sequence_or_point)
    return result

def index_of_first_local_maximum(sequence):
        first_time = True
        index = 0
        for element in sequence:
            if first_time:
                previous = element
                first_time = False
                continue
            if element <= previous:
                return index
            previous = element
            index += 1
        return None

class Function(object):
    _domain = None
    _range = None
    _function_undefined = None

    def __init__(self, function_undefined=None, domain=None):
        if function_undefined is not None:
            self.function_undefined = function_undefined
        if domain is not None:
            if not hasattr(domain, '__iter__') or not hasattr(domain, '__getitem__'):
                    raise TypeError("'domain' should be iterable and support indexing")
            self._domain = domain

    def call_on_single_point(self, x):
        """
        to override, __call__ invokes this to handle both points and sequences
        """
        return 0

    def derivative(self, point):
        return None

    def _check_domain_for(self, feature_name):
        if self.domain is None:
            raise TypeError("'{0}' object does not support {1}, 'domain' should be specified".format(
                self.__class__.__name__, feature_name))

    def plot(self, plt=None):
        self._check_domain_for('plotting')
        if plt is None:
            import matplotlib.pyplot as plt
        plt.plot(self.domain, self.range)
        return plt

    @property
    def function_undefined(self):
        return self._function_undefined

    @function_undefined.setter
    def function_undefined(self, value):
        if value is not None and not isinstance(value, Function):
            raise TypeError("'function_undefined' should be of type 'Function'")
        self._function_undefined = value

    @property
    def domain(self):
        return self._domain

    @property
    def range(self):
        return self()

    def __call__(self, x=None):
        if x is None:
            self._check_domain_for("call with 'None'")
            x = self.domain

        return invoke_method_on(self.call_on_single_point, x)

    def __getitem__(self, index):
        self._check_domain_for('indexing')
        return self.range[index]

    def __len__(self):
        self._check_domain_for('len()')
        return len(self.range)

    def __iter__(self):
        self._check_domain_for('iter()')
        return iter(self.range)

    def __reversed__(self):
        self._check_domain_for('reversed()')
        return reversed(self.range)

class FunctionLinear(Function):
    def __init__(self, a=None, b=None, x_0=None, y_0=None, x_1=None, y_1=None):
        #(x_0, y_0), (x_1, y_1) = sorted([(x_0, y_0), (x_1, y_1)])
        if (a, b) == (None, None):
            a = (float(y_1) - y_0) / (x_1 - x_0)
            b = y_0 - a * x_0
            if isnan(a) or isnan(b):
                pass
        self.a = a
        self.b = b

    def call_on_single_point(self, x):
        return float(self.a * x + self.b)

    def intersect(self, other):
        if almost_equals(self.a, other.a):
            return None
        x = (float(other.b) - self.b) / (self.a - other.a)
        return x, self(x)

    def integral(self, start, end):
        if start >= end:
            return 0
        if self.a == 0:
            return self.b * (end - start)

        x_intercept = self.x_intercept

        if start > x_intercept or end < x_intercept or almost_equals(end, x_intercept) or almost_equals(start, x_intercept):
            return (self(start) + self(end)) * (end - start) / 2.0

        minus_triangle = (x_intercept - start) * self(start)
        plus_triangle = (end - x_intercept) * self(end)
        return minus_triangle + plus_triangle

    def derivative(self, point):
        return self.a

    @property
    def x_intercept(self):
        return - float(self.b) / self.a

    @property
    def y_intercept(self):
        return self(0)

class FunctionHorizontalLinear(FunctionLinear):
    def __init__(self, y_intercept):
        FunctionLinear.__init__(self, a=0, b=y_intercept)

    def call_on_single_point(self, x):
        return self.b

    def integral(self, start, end):
        if start >= end:
            return 0
        if almost_equals(self.b, 0):
            return 0
        return float(self.b) * (end - start)

    def derivative(self, point):
        return 0

FUNCTION_ZERO = FunctionHorizontalLinear(0)
FUNCTION_ONE = FunctionHorizontalLinear(1)

class FunctionComposite(Function):
    is_normalised = False

    def __init__(self, dictionary_bounds_function, function_undefined=None, domain=None, is_normalised=False):
        if is_normalised is not False:
            self.is_normalised = True

        Function.__init__(self, function_undefined=function_undefined, domain=domain)
        if not isinstance(dictionary_bounds_function, dict):
            raise TypeError("'dictionary_bounds_function' should be a dictionary with (lower_bound, higher_bound) "
                            "tuple keys and values of type 'Function'")
        self._dictionary_bounds_function = dictionary_bounds_function

    def call_on_single_point(self, x):
        for function_bounds in self.dictionary_bounds_function:
            (a, b) = function_bounds
            if a <= x:
                if b >= x:
                    if self.dictionary_bounds_function[function_bounds] is None:
                        return None
                    return self.dictionary_bounds_function[function_bounds](x)
        return self.function_undefined(x)

    def integral(self, start, end):
        if self.is_normalised and self.domain is not None:
            if (start < self.domain[0] or almost_equals(start, self.domain[0])) and (
                    end > self.domain[-1] or almost_equals(end, self.domain[-1])):
                return 1.0

        if start >= end:
            return 0
        result = 0
        for function_bounds in self.dictionary_bounds_function:
            (a, b) = function_bounds
            if a <= start:
                if b >= end:
                    return self.dictionary_bounds_function[function_bounds].integral(start, end)
            not_ordered = {
                (start, 0): 's', (end, 0): 'e',
                (a, 1): 'a', (b, 1): 'b'
            }
            order = ''.join([not_ordered[i] for i in sorted(not_ordered)])
            if (a == start or a == end) and order == 'saeb' or (b == start or b == end) and order == 'asbe':
                continue
            if order in 'seab abse':
                continue
            if order == 'saeb':
                b = end
            elif order == 'asbe':
                a = start
            result += self.dictionary_bounds_function[function_bounds].integral(a, b)
        return result

    def find_bounds_for(self, point):
        for bounds in self.dictionary_bounds_function:
            (a, b) = bounds
            if a <= point and b >= point:
                return bounds

    def derivative(self, point):
        return self.dictionary_bounds_function[self.find_bounds_for(point)].derivative(point)

    def function_in_point(self, point):
        for bounds in self.dictionary_bounds_function:
            a, b = bounds
            if a <= point <= b:
                return self.dictionary_bounds_function[bounds]
        return None

    # def functions_in_interval(self, interval_start, interval_end):
    #     dictionary_bounds_function = {}
    #     for bounds in self.dictionary_bounds_function:
    #         a, b = bounds
    #         if (interval_start < a or almost_equals(interval_start, a)) and (
    #
    #         ):

    @property
    def dictionary_bounds_function(self):
        return self._dictionary_bounds_function

class FunctionPiecewiseLinear(FunctionComposite):
    def __init__(self, dictionary_input_output, function_undefined=None, is_normalised=False):
        self.input_list, self.output_list = convert_dict_to_sorted_lists(dictionary_input_output)

        dictionary_bounds_function = {}
        for i in xrange(1, len(self.input_list)):
            x_0, x_1 = self.input_list[i - 1], self.input_list[i]
            y_0, y_1 = self.output_list[i - 1], self.output_list[i]
            dictionary_bounds_function[(x_0, x_1)] = FunctionLinear(x_0=x_0, x_1=x_1, y_0=y_0, y_1=y_1)
        if NEGATIVE_INFINITY not in self.input_list:
            dictionary_bounds_function[(NEGATIVE_INFINITY, self.input_list[0])] = function_undefined
        if POSITIVE_INFINITY not in self.input_list:
            dictionary_bounds_function[(self.input_list[-1], POSITIVE_INFINITY)] = function_undefined
        FunctionComposite.__init__(self, dictionary_bounds_function,
                                   function_undefined=function_undefined,
                                   domain=self.input_list,
                                   is_normalised=is_normalised)

    def normalised(self):
        area = self.integral(NEGATIVE_INFINITY, POSITIVE_INFINITY)
        if almost_equals(area, 0):
            area = self.integral(NEGATIVE_INFINITY, POSITIVE_INFINITY)
        dictionary_input_output = {}
        output_list = [y / area for y in self.output_list]
        for i in xrange(len(self.input_list)):
            dictionary_input_output[self.input_list[i]] = output_list[i]
        result = FunctionPiecewiseLinear(dictionary_input_output, function_undefined=self.function_undefined)
        result.is_normalised = True
        return result

    def __and__(self, other):
        for bounds in self.dictionary_bounds_function:
            a, b = bounds
            linear_function = self.dictionary_bounds_function[bounds]

if __name__ == '__main__':
    a = FunctionLinear(1, 0)
    b = FunctionLinear(-1, 1)
    print a.intersect(b)


"""
Created on Apr 24, 2013

@author: agross
"""

import numpy as np
import pandas as pd
import matplotlib.pylab as plt

import Stats.Scipy as Stats
from Figures.FigureHelpers import latex_float, init_ax
from Figures.FigureHelpers import prettify_ax
from Helpers.Pandas import match_series, true_index
colors = plt.rcParams['axes.color_cycle'] * 10

def _violin_plot(ax, data, pos=[], bp=False):
    """
    http://pyinsci.blogspot.com/2009/09/violin-plot-with-matplotlib.html

    Create violin plots on an axis.  Internal to module as it does not
    use Pandas data-structures.  This is split off due to it's being a
    reuse of the code from the blog-post linked above, and I wanted to keep
    the original code untouched.
    """
    from scipy.stats import gaussian_kde
    from numpy import arange

    # dist = max(pos)-min(pos)
    dist = len(pos)
    w = min(0.25 * max(dist, 1.0), 0.5)
    for p, d in enumerate(data):
        try:
            k = gaussian_kde(d)  # calculates the kernel density
            m = k.dataset.min()  # lower bound of violin
            M = k.dataset.max()  # upper bound of violin
            x = arange(m, M, (M - m) / 100.)  # support for violin
            v = k.evaluate(x)  # violin profile (density curve)
            v = v / v.max() * w  # scaling the violin to the available space
            ax.fill_betweenx(x, p, v + p, facecolor='y', alpha=0.1)
            ax.fill_betweenx(x, p, -v + p, facecolor='y', alpha=0.1)
        except:
            pass
    if bp:
        box_plot = ax.boxplot(data, notch=1, positions=range(len(pos)), vert=1,
                             widths=.25)
        return box_plot

def box_plot_pandas(bin_vec, real_vec, ax=None, order=None):
    """
    Wrapper around matplotlib's boxplot function.

    Inputs
        bin_vec: Series of labels
        real_vec: Series of measurements to be grouped according to bin_vec
    """
    _, ax = init_ax(ax)
    bin_vec, real_vec = match_series(bin_vec, real_vec)
    if order is not None:
        categories = order
    else:
        categories = bin_vec.value_counts().index
    data = [real_vec[bin_vec == num] for num in categories]
    bp = ax.boxplot(data, positions=range(len(categories)), widths=.3,
                    patch_artist=True)
    if real_vec.name:
        ax.set_ylabel(real_vec.name)
    if bin_vec.name:
        ax.set_xlabel(bin_vec.name)
    ax.set_xticklabels(categories)
    [p.set_visible(False) for p in bp['fliers']]
    [p.set_visible(False) for p in bp['caps']]
    [p.set_visible(False) for p in bp['whiskers']]
    for p in bp['medians']:
        p.set_color(colors[0])
        p.set_lw(3)
        p.set_alpha(.8)
    for i, p in enumerate(bp['boxes']):
        p.set_color('grey')
        p.set_lw(3)
        p.set_alpha(.7)
        if len(data[i]) < 3:
            p.set_alpha(0)

def violin_plot_pandas(bin_vec, real_vec, ann='p', order=None, ax=None,
                       filename=None):
    """
    http://pyinsci.blogspot.com/2009/09/violin-plot-with-matplotlib.html
    Wrapper around matplotlib's boxplot function to add violin profile.

    Inputs
        bin_vec: Series of labels
        real_vec: Series of measurements to be grouped according to bin_vec
    """
    fig, ax = init_ax(ax)
    ax.set_ylabel(real_vec.name)
    ax.set_xlabel(bin_vec.name)
    bin_vec, real_vec = match_series(bin_vec, real_vec)
    try:
        if order is None:
            categories = bin_vec.value_counts().index
        else:
            categories = order
        _violin_plot(ax, [real_vec[bin_vec == num] for num in categories],
                     pos=categories, bp=True)
        ax.set_xticklabels([str(c) + '\n(n=%i)' % sum(bin_vec == c)
                            for c in categories])
    except:
        box_plot_pandas(bin_vec, real_vec, ax=ax)

    #if type(bin_vec.name) == str:
    #    ax.set_title(str(bin_vec.name) + ' x ' + str(real_vec.name))

    p_value = Stats.kruskal_pandas(bin_vec, real_vec)['p']
    if ann == 'p_fancy':
        ax.annotate('$p = {}$'.format(latex_float(p_value)), (.95, -.02),
                    xycoords='axes fraction', ha='right', va='bottom', size=14)
    if ann == 'p':
        ax.annotate('p = {0:.1e}'.format(p_value), (.95, .02),
                    xycoords='axes fraction', ha='right', va='bottom', size=12)
    elif ann is not None:
        ax.annotate(ann, (.95, .02), xycoords='axes fraction', ha='right',
                    va='bottom', size=12)
    if filename is not None:
        fig.savefig(filename)
    return

def violin_plot_series(s, **kw_args):
    """
    Wrapper for drawing a violin plot on a series with a multi-index.
    The second level of the index is used as the binning variable.
    """
    assert s.index.levshape[1] > 1
    violin_plot_pandas(pd.Series(s.index.get_level_values(1), s.index), s,
                       **kw_args)

def paired_boxplot_o(boxes):
    """
    Wrapper around plt.boxplot to draw paired boxplots
    for a set of boxes.

    Input is the same as plt.boxplot:
        Array or a sequence of vectors.
    """
    fig = plt.figure(figsize=(len(boxes) / 2.5, 4))
    ax1 = fig.add_subplot(111)
    plt.subplots_adjust(left=0.075, right=0.95, top=0.9, bottom=0.25)
    bp = ax1.boxplot(boxes, notch=0, positions=np.arange(len(boxes)) +
                     1.5 * (np.arange(len(boxes)) / 2), patch_artist=True)
    [p.set_color(colors[0]) for p in bp['boxes'][::2]]
    [p.set_color('black') for p in bp['whiskers']]
    [p.set_color('black') for p in bp['fliers']]
    [p.set_alpha(.4) for p in bp['fliers']]
    [p.set_alpha(.6) for p in bp['boxes']]
    [p.set_edgecolor('black') for p in bp['boxes']]
    ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                  alpha=0.5)

    # Hide these grid behind plot objects
    ax1.set_axisbelow(True)
    ax1.set_ylabel('$Log_{2}$ RNA Expression')
    ax1.set_xticks(3.5 * np.arange(len(boxes) / 2) + .5)
    return ax1, bp

def paired_boxplot(boxes, ax1=None):
    if not ax1:
        fig = plt.figure(figsize=(len(boxes) / 2.5, 4))
        ax1 = fig.add_subplot(111)
    plt.subplots_adjust(left=0.075, right=0.95, top=0.9, bottom=0.25)
    bp = ax1.boxplot(boxes, notch=0, positions=np.arange(len(boxes)) +
                     1.5 * (np.arange(len(boxes)) / 2), patch_artist=True)
    [p.set_color(colors[0]) for p in bp['boxes'][::2]]
    [p.set_color(colors[1]) for p in bp['boxes'][1::2]]
    [p.set_color('black') for p in bp['whiskers']]
    [p.set_color('black') for p in bp['fliers']]
    [p.set_alpha(.4) for p in bp['fliers']]
    [p.set_alpha(.8) for p in bp['boxes']]
    [p.set_edgecolor('black') for p in bp['boxes']]
    ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                  alpha=0.5)

    # Hide these grid behind plot objects
    ax1.set_axisbelow(True)
    ax1.set_ylabel('$Log_{2}$ RNA Expression')
    ax1.set_xticks(3.5 * np.arange(len(boxes) / 2) + .5)
    return ax1, bp

def paired_boxplot_tumor_normal(df, sig=True, cutoffs=[.01, .00001],
                                order=None, ax=None):
    """
    Draws a paired boxplot given a DataFrame with both tumor and normal
    samples on the index. '01' and '11' are hard-coded as the ids for
    tumor/normal.
    """
    n = df.groupby(level=0).size() == 2
    df = df.ix[n[n].index]
    if order is None:
        o = df.xs('11', level=1).median().order().index
        df = df[o[::-1]]
    else:
        df = df[order]
    l1 = list(df.xs('01', level=1).as_matrix().T)
    l2 = list(df.xs('11', level=1).as_matrix().T)
    boxes = [x for t in zip(l1, l2) for x in t]
    ax1, bp = paired_boxplot(boxes, ax)

    test = lambda v: Stats.ttest_rel(v.unstack()['01'], v.unstack()['11'])
    res = df.apply(test).T
    p = res.p

    if sig:
        pts = [(i * 3.5 + .5, 18) for i, n in enumerate(p) if n < cutoffs[1]]
        if len(pts) > 0:
            s1 = ax1.scatter(*zip(*pts), marker='$**$', label='$p<10^{-5}$', s=200)
        else:
            s1 = None
        pts = [(i * 3.5 + .5, 18) for i, n in enumerate(p)
               if (n < cutoffs[0]) and (n > cutoffs[1])]
        if len(pts) > 0:
            s2 = ax1.scatter(*zip(*pts), marker='$*$', label='$p<10^{-2}$', s=30)
        else:
            s2 = None
        ax1.legend(bp['boxes'][:2] + [s2, s1],
                   ('Tumor', 'Normal', '$p<10^{-2}$', '$p<10^{-5}$'),
                   loc='best', scatterpoints=1)
    else:
        ax1.legend(bp['boxes'][:2], ('Tumor', 'Normal'), loc='best')
    ax1.set_xticklabels(df.columns)

def boxplot_panel(hit_vec, response_df):
    """
    Draws a series of paired boxplots with the rows of the response_df
    split according to hit_vec.
    """
    b = response_df.copy()
    b.columns = pd.MultiIndex.from_arrays([b.columns, hit_vec.ix[b.columns]])
    b = b.T
    v1, v2 = hit_vec.unique()
    test = lambda v: Stats.anova(v.reset_index(level=1)[v.index.names[1]],
                                 v.reset_index(level=1)[v.name])
    res = b.apply(test).T
    p = res.p.order()
    b = b.ix[:, p.index]

    l1 = list(b.xs(v1, level=1).as_matrix().T)
    l2 = list(b.xs(v2, level=1).as_matrix().T)

    boxes = [x for t in zip(l1, l2) for x in t]
    ax1, bp = paired_boxplot(boxes)

    y_lim = (response_df.T.quantile(.9).max()) * 1.2
    pts = [(i * 3.5 + .5, y_lim) for i, n in enumerate(p) if n < .00001]
    if len(pts) > 0:
        s1 = ax1.scatter(*zip(*pts), marker='$**$', label='$p<10^{-5}$', s=200)
    else:
        s1 = None
    pts = [(i * 3.5 + .5, y_lim) for i, n in enumerate(p) if (n < .01)
           and (n > .00001)]
    if len(pts) > 0:
        s2 = ax1.scatter(*zip(*pts), marker='$*$', label='$p<10^{-2}$', s=30)
    else:
        s2 = None
    ax1.set_xticklabels(b.columns)
    ax1.legend(bp['boxes'][:2] + [s2, s1],
               (v1, v2, '$p<10^{-2}$', '$p<10^{-5}$'),
               loc='best', scatterpoints=1)

def paired_bp_tn_split(vec, assignment, ax=None, split_vals=('01', '11'),
                       data_type='gene expression'):
    """
    Paired boxplot for a single Series, with splitting on the index,
    grouped by assignment. I.E. Tumor-Normal gene expression split by
    cancer.

    vec:
        vector of values to plot.
    assignment:
        vector mapping keys to group assignment
    ax (None):
        matplotlib axis to plot on or None
    split_vals ('01','11'):
        Values to split the boxplot pairing on. The default of
        ('01','11') indicates tumor vs. normal in the standard
        TCGA barcode nomenclature.  This should coorespond to values
        on the second level of the index for vec and assignment.

    **both vec and assignment should have an overlapping index with
    multiple levels**
    """
    _, ax = init_ax(ax, figsize=(8, 3))
    if vec.name != None:
        label = vec.name  # lose label in manipulation
    else:
        label = ''
    g1 = split_vals[0]
    g2 = split_vals[1]
    vec = pd.concat([vec[:, g1], vec[:, g2]], keys=[g1, g2],
                    axis=1)
    vec = vec.dropna().stack()

    counts = vec.unstack().groupby(assignment).size()
    groups = list(true_index(counts > 5))
    groups = vec.unstack().groupby(assignment).median()[g1].ix[groups]
    groups = groups.order().index[::-1]

    l1 = [np.array(vec[:, g1].ix[true_index(assignment == c)].dropna())
          for c in groups]
    l2 = [np.array(vec[:, g2].ix[true_index(assignment == c)].dropna())
          for c in groups]
    boxes = [x for t in zip(l1, l2) for x in t if len(t[1]) > 5]

    ax, bp = paired_boxplot(boxes, ax)
    labels = ['{}\n({})'.format(c, counts[c]) for c in groups]
    ax.set_xticklabels(labels)
    prettify_ax(ax)
    ax.set_ylabel('{} {}'.format(label, data_type))


#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""Example of DNNClassifier for Iris plant dataset, with pipeline."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from sklearn import cross_validation
from sklearn.datasets import load_iris
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
import tensorflow as tf

from tensorflow.contrib import learn

def main(unused_argv):
  iris = load_iris()
  x_train, x_test, y_train, y_test = cross_validation.train_test_split(
      iris.data, iris.target, test_size=0.2, random_state=42)

  # It's useful to scale to ensure Stochastic Gradient Descent
  # will do the right thing.
  scaler = StandardScaler()

  # DNN classifier.
  classifier = learn.DNNClassifier(
      feature_columns=learn.infer_real_valued_columns_from_input(x_train),
      hidden_units=[10, 20, 10], n_classes=3)

  pipeline = Pipeline([('scaler', scaler),
                       ('DNNclassifier', classifier)])

  pipeline.fit(x_train, y_train, DNNclassifier__steps=200)

  score = accuracy_score(y_test, pipeline.predict(x_test))
  print('Accuracy: {0:f}'.format(score))

if __name__ == '__main__':
  tf.app.run()


import os

import numpy as np
import pandas as pd
from pandas.tools.plotting import scatter_matrix
import pylab
import statsmodels.formula.api as smf
import statsmodels.stats.api as sms

from industry_to_days import get_census_mapper

"""
USAGE:
python full_2011.py

CREATES:
results/2011/clean.csv
results/2011/corr.txt
results/2011/het_breushpagan.txt
results/2011/ols1.txt
results/2011/ols2.txt
results/2011/scatter_matrix.png
results/2011/summary.txt
"""

COL_ORDER = ['vacation', 'paid_vacation', 'age', 'fam_size', 'is_female',
             'income10', 'salary', 'is_employed']

PSID_CSV = os.path.abspath(
        os.path.join(
            os.path.dirname(__file__),
            'psid', '2011.csv'))

def get_f_path(fname):
    return os.path.abspath(
            os.path.join(
                os.path.dirname(__file__),
                'results', '2011', fname))

CLEAN_CSV = get_f_path('clean.csv')
CORR_TXT = get_f_path('corr.txt')
HET_BP_TXT = get_f_path('het_breushpagan.txt')
OLS1_TXT = get_f_path('ols1.txt')
OLS2_TXT = get_f_path('ols2.txt')
SCAT_MATRIX_PNG = get_f_path('scatter_matrix.png')
SUMMARY_TXT = get_f_path('summary.txt')

f_exists = (lambda file_: os.path.isfile(file_))

def _calc_vacation(row):
    took, days, weeks, months = (row['took_vac'], row['days_vac'],
                                 row['weeks_vac'], row['months_vac'])
    if took in [8, 9] or (days in [998, 999]) or (months in [98, 99]) or (
        weeks in [98, 99]):
        return np.nan
    elif took == 5:
        return 0
    else:
        return days + (5 * weeks) + (22 * months)

def _calc_salary(row):
    amt, unit = row['salary_amt'], row['salary_unit']
    if amt in [0.0, 9999998.0] or unit in [0, 7, 8, 9]:
        return np.nan
    if unit == 3:  # one week
        scalar = 52.0
    elif unit == 4:  # two weeks
        scalar = 26.0
    elif unit == 5:  # one month
        scalar = 12.0
    elif unit == 6:  # one year
        scalar = 1.0
    return scalar * amt

def clean(df):
    # make sex into dummy for is_female
    df['is_female'] = df['sex'] - 1
    # remove unknown age values
    df.age = df.age.replace(999, np.nan)
    # figure out total vacation taken
    df['vacation'] = df.apply(_calc_vacation, axis=1)
    # fix salary to be annual amount
    df['salary'] = df.apply(_calc_salary, axis=1)
    # remove outliers
    df.ix[df.salary < 1e3] = np.nan
    df.ix[df.salary >= 400e3] = np.nan
    df.ix[df.income10 < 1e3] = np.nan
    df.ix[df.income10 >= 400e3] = np.nan
    # make employment into dummy for is_employed
    df['is_employed'] = df.employment
    # remove all those not working
    for i in range(2,10) + [99]:
        df.is_employed.replace(i, 0, inplace=True)
    # merge industry data
    df['paid_vacation'] = df.industry.map(get_census_mapper())
    # drop old values
    for col in ['took_vac', 'days_vac', 'weeks_vac', 'months_vac', 'industry',
                'salary_amt', 'salary_unit', 'sex', 'employment']:
        df.drop(col, axis=1, inplace=True)
    df = df.reindex_axis(sorted(df.columns, key=COL_ORDER.index), axis=1)
    return df

def do_stats(df):
    # Only view those that received vacation and are employed
    df.is_employed.replace(0.0, np.nan, inplace=True)
    df.paid_vacation.replace(0.0, np.nan, inplace=True)
    df.dropna(inplace=True)
    # No longer need this dummy
    df.drop('is_employed', axis=1, inplace=True)

    # Summary stats
    if not f_exists(SUMMARY_TXT):
        summary = df.describe().T
        summary = np.round(summary, decimals=3)
        with open(SUMMARY_TXT, 'w') as f:
            f.write(summary.to_string())

    # Test for autocorrelation: scatter matrix, correlation, run OLS
    if not f_exists(SCAT_MATRIX_PNG):
        scatter_matrix(df, alpha=0.2, figsize=(64, 64), diagonal='hist')
        pylab.savefig(SCAT_MATRIX_PNG, bbox_inches='tight')
    if not f_exists(CORR_TXT):
        corr = df.corr()
        corr = corr.reindex_axis(
            sorted(corr.columns, key=COL_ORDER.index), axis=0)
        corr = corr.reindex_axis(
            sorted(corr.columns, key=COL_ORDER.index), axis=1)
        for i, k in enumerate(corr):
            row = corr[k]
            for j in range(len(row)):
                if j > i:
                    row[j] = np.nan
        with open(CORR_TXT, 'w') as f:
            f.write(np.round(corr, decimals=3).to_string(na_rep=''))
    if not f_exists(OLS1_TXT):
        ols_results = smf.ols(
            formula='vacation ~ paid_vacation + np.square(paid_vacation) + '
                    'age + fam_size + is_female + income10 + salary + '
                    'np.square(salary)',
            data=df).fit()
        with open(OLS1_TXT, 'w') as f:
            f.write(str(ols_results.summary()))
            f.write('\n\nCondition Number: {}'.format(
                np.linalg.cond(ols_results.model.exog)))

    # Need to drop salary, too much autocorrelation
    df.drop('salary', axis=1, inplace=True)

    # Test for autocorrelation: scatter matrix, correlation, run OLS
    if not f_exists(HET_BP_TXT):
        ols_results = smf.ols(
            formula='vacation ~ paid_vacation + np.square(paid_vacation) + '
                    'age + fam_size + is_female + income10',
            data=df).fit()
        names = ['LM', 'LM P val.', 'F Stat.', 'F Stat. P val.']
        test = sms.het_breushpagan(ols_results.resid, ols_results.model.exog)
        f_p = test[3]
        with open(HET_BP_TXT, 'w') as f:
            str_ =  '\n'.join('{}: {}'.format(n, v)
                              for n, v in zip(names, test))
            f.write(str_ + '\n\n')
            if f_p < .01:
                f.write('No Heteroskedasticity found.\n')
            else:
                f.write('Warning: Heteroskedasticity found!\n')

    # no Heteroskedasticity found
    # final OLS results with robust standard errors
    if not f_exists(OLS2_TXT):
        ols_results = smf.ols(
            formula='vacation ~ paid_vacation + np.square(paid_vacation) + '
                    'age + fam_size + is_female + income10',
            data=df).fit().get_robustcov_results(cov_type='HAC', maxlags=1)
        with open(OLS2_TXT, 'w') as f:
            f.write(str(ols_results.summary()))
            f.write('\n\nCondition Number: {}'.format(
                np.linalg.cond(ols_results.model.exog)))
    return df

def main():
    df = None
    if f_exists(CLEAN_CSV):
        df = pd.io.parsers.read_csv(CLEAN_CSV)
        df.drop('Unnamed: 0', axis=1, inplace=True)
    else:
        with open(PSID_CSV) as csv:
            df = pd.io.parsers.read_csv(csv)
        df = clean(df)
        # write output to a file
        with open(CLEAN_CSV, 'w+') as csv:
            df.to_csv(path_or_buf=csv)
    return do_stats(df)

if __name__ == '__main__':
    main()
    print '2011 succeeds! :)'


"""
Benchmark scikit-learn's Ward implement compared to SciPy's
"""

import time

import numpy as np
from scipy.cluster import hierarchy
import matplotlib.pyplot as plt

from sklearn.cluster import AgglomerativeClustering

ward = AgglomerativeClustering(n_clusters=3, linkage='ward')

n_samples = np.logspace(.5, 3, 9)
n_features = np.logspace(1, 3.5, 7)
N_samples, N_features = np.meshgrid(n_samples,
                                    n_features)
scikits_time = np.zeros(N_samples.shape)
scipy_time = np.zeros(N_samples.shape)

for i, n in enumerate(n_samples):
    for j, p in enumerate(n_features):
        X = np.random.normal(size=(n, p))
        t0 = time.time()
        ward.fit(X)
        scikits_time[j, i] = time.time() - t0
        t0 = time.time()
        hierarchy.ward(X)
        scipy_time[j, i] = time.time() - t0

ratio = scikits_time / scipy_time

plt.figure("scikit-learn Ward's method benchmark results")
plt.imshow(np.log(ratio), aspect='auto', origin="lower")
plt.colorbar()
plt.contour(ratio, levels=[1, ], colors='k')
plt.yticks(range(len(n_features)), n_features.astype(np.int))
plt.ylabel('N features')
plt.xticks(range(len(n_samples)), n_samples.astype(np.int))
plt.xlabel('N samples')
plt.title("Scikit's time, in units of scipy time (log)")
plt.show()