import functools
import numpy as np
import pandas as pd

from . import ops
from .combine import concat
from .common import (
    ImplementsArrayReduce, ImplementsDatasetReduce, _maybe_promote,
)
from .pycompat import zip
from .utils import peek_at, maybe_wrap_array, safe_cast_to_index
from .variable import as_variable, Variable, Coordinate

def unique_value_groups(ar):
    """Group an array by its unique values.

    Parameters
    ----------
    ar : array-like
        Input array. This will be flattened if it is not already 1-D.

    Returns
    -------
    values : np.ndarray
        Sorted, unique values as returned by `np.unique`.
    indices : list of lists of int
        Each element provides the integer indices in `ar` with values given by
        the corresponding value in `unique_values`.
    """
    inverse, values = pd.factorize(ar, sort=True)
    groups = [[] for _ in range(len(values))]
    for n, g in enumerate(inverse):
        if g >= 0:
            # pandas uses -1 to mark NaN, but doesn't include them in values
            groups[g].append(n)
    return values, groups

def _get_fill_value(dtype):
    """Return a fill value that appropriately promotes types when used with
    np.concatenate
    """
    dtype, fill_value = _maybe_promote(dtype)
    return fill_value

def _dummy_copy(xarray_obj):
    from .dataset import Dataset
    from .dataarray import DataArray
    if isinstance(xarray_obj, Dataset):
        res = Dataset(dict((k, _get_fill_value(v.dtype))
                           for k, v in xarray_obj.data_vars.items()),
                      dict((k, _get_fill_value(v.dtype))
                           for k, v in xarray_obj.coords.items()
                           if k not in xarray_obj.dims),
                      xarray_obj.attrs)
    elif isinstance(xarray_obj, DataArray):
        res = DataArray(_get_fill_value(xarray_obj.dtype),
                        dict((k, _get_fill_value(v.dtype))
                             for k, v in xarray_obj.coords.items()
                             if k not in xarray_obj.dims),
                        name=xarray_obj.name,
                        attrs=xarray_obj.attrs)
    else:  # pragma: no cover
        raise AssertionError
    return res

class GroupBy(object):
    """A object that implements the split-apply-combine pattern.

    Modeled after `pandas.GroupBy`. The `GroupBy` object can be iterated over
    (unique_value, grouped_array) pairs, but the main way to interact with a
    groupby object are with the `apply` or `reduce` methods. You can also
    directly call numpy methods like `mean` or `std`.

    You should create a GroupBy object by using the `DataArray.groupby` or
    `Dataset.groupby` methods.

    See Also
    --------
    Dataset.groupby
    DataArray.groupby
    """
    def __init__(self, obj, group, squeeze=False, grouper=None):
        """Create a GroupBy object

        Parameters
        ----------
        obj : Dataset or DataArray
            Object to group.
        group : DataArray or Coordinate
            1-dimensional array with the group values.
        squeeze : boolean, optional
            If "group" is a coordinate of object, `squeeze` controls whether
            the subarrays have a dimension of length 1 along that coordinate or
            if the dimension is squeezed out.
        grouper : pd.Grouper, optional
            Used for grouping values along the `group` array.
        """
        from .dataset import as_dataset

        if group.ndim != 1:
            # TODO: remove this limitation?
            raise ValueError('`group` must be 1 dimensional')
        if getattr(group, 'name', None) is None:
            raise ValueError('`group` must have a name')
        if not hasattr(group, 'dims'):
            raise ValueError("`group` must have a 'dims' attribute")
        group_dim, = group.dims

        try:
            expected_size = obj.dims[group_dim]
        except TypeError:
            expected_size = obj.shape[obj.get_axis_num(group_dim)]
        if group.size != expected_size:
            raise ValueError('the group variable\'s length does not '
                             'match the length of this variable along its '
                             'dimension')
        full_index = None

        if grouper is not None:
            # time-series resampling
            index = safe_cast_to_index(group)
            if not index.is_monotonic:
                # TODO: sort instead of raising an error
                raise ValueError('index must be monotonic for resampling')
            s = pd.Series(np.arange(index.size), index)
            first_items = s.groupby(grouper).first()
            if first_items.isnull().any():
                full_index = first_items.index
                first_items = first_items.dropna()
            bins = first_items.values.astype(np.int64)
            group_indices = ([slice(i, j) for i, j in zip(bins[:-1], bins[1:])] +
                             [slice(bins[-1], None)])
            unique_coord = Coordinate(group.name, first_items.index)
        elif group.name in obj.dims:
            # assume that group already has sorted, unique values
            if group.dims != (group.name,):
                raise ValueError('`group` is required to be a coordinate if '
                                 '`group.name` is a dimension in `obj`')
            group_indices = np.arange(group.size)
            if not squeeze:
                # group_indices = group_indices.reshape(-1, 1)
                # use slices to do views instead of fancy indexing
                group_indices = [slice(i, i + 1) for i in group_indices]
            unique_coord = group
        else:
            # look through group to find the unique values
            unique_values, group_indices = unique_value_groups(group)
            unique_coord = Coordinate(group.name, unique_values)

        self.obj = obj
        self.group = group
        self.group_dim = group_dim
        self.group_indices = group_indices
        self.unique_coord = unique_coord
        self._groups = None
        self._full_index = full_index

    @property
    def groups(self):
        # provided to mimic pandas.groupby
        if self._groups is None:
            self._groups = dict(zip(self.unique_coord.values,
                                    self.group_indices))
        return self._groups

    def __len__(self):
        return self.unique_coord.size

    def __iter__(self):
        return zip(self.unique_coord.values, self._iter_grouped())

    def _iter_grouped(self):
        """Iterate over each element in this group"""
        for indices in self.group_indices:
            yield self.obj.isel(**{self.group_dim: indices})

    def _infer_concat_args(self, applied_example):
        if self.group_dim in applied_example.dims:
            concat_dim = self.group
            positions = self.group_indices
        else:
            concat_dim = self.unique_coord
            positions = None
        return concat_dim, positions

    @staticmethod
    def _binary_op(f, reflexive=False, **ignored_kwargs):
        @functools.wraps(f)
        def func(self, other):
            g = f if not reflexive else lambda x, y: f(y, x)
            applied = self._yield_binary_applied(g, other)
            combined = self._concat(applied)
            return combined
        return func

    def _yield_binary_applied(self, func, other):
        dummy = None

        for group_value, obj in self:
            try:
                other_sel = other.sel(**{self.group.name: group_value})
            except AttributeError:
                raise TypeError('GroupBy objects only support binary ops '
                                'when the other argument is a Dataset or '
                                'DataArray')
            except KeyError:
                if self.group.name not in other.dims:
                    raise ValueError('incompatible dimensions for a grouped '
                                     'binary operation: the group variable %r '
                                     'is not a dimension on the other argument'
                                     % self.group.name)
                if dummy is None:
                    dummy = _dummy_copy(other)
                other_sel = dummy

            result = func(obj, other_sel)
            yield result

    def _maybe_restore_empty_groups(self, combined):
        """Our index contained empty groups (e.g., from a resampling). If we
        reduced on that dimension, we want to restore the full index.
        """
        if (self._full_index is not None and self.group.name in combined.dims):
            indexers = {self.group.name: self._full_index}
            combined = combined.reindex(**indexers)
        return combined

    def fillna(self, value):
        """Fill missing values in this object by group.

        This operation follows the normal broadcasting and alignment rules that
        xarray uses for binary arithmetic, except the result is aligned to this
        object (``join='left'``) instead of aligned to the intersection of
        index coordinates (``join='inner'``).

        Parameters
        ----------
        value : valid type for the grouped object's fillna method
            Used to fill all matching missing values by group.

        Returns
        -------
        same type as the grouped object

        See also
        --------
        Dataset.fillna
        DataArray.fillna
        """
        return self._fillna(value)

    def where(self, cond):
        """Return an object of the same shape with all entries where cond is
        True and all other entries masked.

        This operation follows the normal broadcasting and alignment rules that
        xarray uses for binary arithmetic.

        Parameters
        ----------
        cond : DataArray or Dataset

        Returns
        -------
        same type as the grouped object

        See also
        --------
        Dataset.where
        """
        return self._where(cond)

    def _first_or_last(self, op, skipna, keep_attrs):
        if isinstance(self.group_indices[0], (int, np.integer)):
            # NB. this is currently only used for reductions along an existing
            # dimension
            return self.obj
        return self.reduce(op, self.group_dim, skipna=skipna,
                           keep_attrs=keep_attrs, allow_lazy=True)

    def first(self, skipna=None, keep_attrs=True):
        """Return the first element of each group along the group dimension
        """
        return self._first_or_last(ops.first, skipna, keep_attrs)

    def last(self, skipna=None, keep_attrs=True):
        """Return the last element of each group along the group dimension
        """
        return self._first_or_last(ops.last, skipna, keep_attrs)

    def assign_coords(self, **kwargs):
        """Assign coordinates by group.

        See also
        --------
        Dataset.assign_coords
        """
        return self.apply(lambda ds: ds.assign_coords(**kwargs))

class DataArrayGroupBy(GroupBy, ImplementsArrayReduce):
    """GroupBy object specialized to grouping DataArray objects
    """
    def _iter_grouped_shortcut(self):
        """Fast version of `_iter_grouped` that yields Variables without
        metadata
        """
        var = self.obj.variable
        for indices in self.group_indices:
            yield var[{self.group_dim: indices}]

    def _concat_shortcut(self, applied, concat_dim, positions):
        # nb. don't worry too much about maintaining this method -- it does
        # speed things up, but it's not very interpretable and there are much
        # faster alternatives (e.g., doing the grouped aggregation in a
        # compiled language)
        stacked = Variable.concat(
            applied, concat_dim, positions, shortcut=True)
        stacked.attrs.update(self.obj.attrs)
        result = self.obj._replace_maybe_drop_dims(stacked)
        result._coords[concat_dim.name] = as_variable(concat_dim, copy=True)
        return result

    def _restore_dim_order(self, stacked):
        def lookup_order(dimension):
            if dimension == self.group.name:
                dimension, = self.group.dims
            if dimension in self.obj.dims:
                axis = self.obj.get_axis_num(dimension)
            else:
                axis = 1e6  # some arbitrarily high value
            return axis

        new_order = sorted(stacked.dims, key=lookup_order)
        return stacked.transpose(*new_order)

    def apply(self, func, shortcut=False, **kwargs):
        """Apply a function over each array in the group and concatenate them
        together into a new array.

        `func` is called like `func(ar, *args, **kwargs)` for each array `ar`
        in this group.

        Apply uses heuristics (like `pandas.GroupBy.apply`) to figure out how
        to stack together the array. The rule is:
        1. If the dimension along which the group coordinate is defined is
           still in the first grouped array after applying `func`, then stack
           over this dimension.
        2. Otherwise, stack over the new dimension given by name of this
           grouping (the argument to the `groupby` function).

        Parameters
        ----------
        func : function
            Callable to apply to each array.
        shortcut : bool, optional
            Whether or not to shortcut evaluation under the assumptions that:
            (1) The action of `func` does not depend on any of the array
                metadata (attributes or coordinates) but only on the data and
                dimensions.
            (2) The action of `func` creates arrays with homogeneous metadata,
                that is, with the same dimensions and attributes.
            If these conditions are satisfied `shortcut` provides significant
            speedup. This should be the case for many common groupby operations
            (e.g., applying numpy ufuncs).
        **kwargs
            Used to call `func(ar, **kwargs)` for each array `ar`.

        Returns
        -------
        applied : DataArray
            The result of splitting, applying and combining this array.
        """
        if shortcut:
            grouped = self._iter_grouped_shortcut()
        else:
            grouped = self._iter_grouped()
        applied = (maybe_wrap_array(arr, func(arr, **kwargs)) for arr in grouped)
        combined = self._concat(applied, shortcut=shortcut)
        result = self._maybe_restore_empty_groups(combined)
        return result

    def _concat(self, applied, shortcut=False):
        # peek at applied to determine which coordinate to stack over
        applied_example, applied = peek_at(applied)
        concat_dim, positions = self._infer_concat_args(applied_example)

        if shortcut:
            combined = self._concat_shortcut(applied, concat_dim, positions)
        else:
            combined = concat(applied, concat_dim, positions=positions)

        if isinstance(combined, type(self.obj)):
            combined = self._restore_dim_order(combined)
        return combined

    def reduce(self, func, dim=None, axis=None, keep_attrs=False,
               shortcut=True, **kwargs):
        """Reduce the items in this group by applying `func` along some
        dimension(s).

        Parameters
        ----------
        func : function
            Function which can be called in the form
            `func(x, axis=axis, **kwargs)` to return the result of collapsing an
            np.ndarray over an integer valued axis.
        dim : str or sequence of str, optional
            Dimension(s) over which to apply `func`.
        axis : int or sequence of int, optional
            Axis(es) over which to apply `func`. Only one of the 'dimension'
            and 'axis' arguments can be supplied. If neither are supplied, then
            `func` is calculated over all dimension for each group item.
        keep_attrs : bool, optional
            If True, the datasets's attributes (`attrs`) will be copied from
            the original object to the new one.  If False (default), the new
            object will be returned without attributes.
        **kwargs : dict
            Additional keyword arguments passed on to `func`.

        Returns
        -------
        reduced : Array
            Array with summarized data and the indicated dimension(s)
            removed.
        """
        def reduce_array(ar):
            return ar.reduce(func, dim, axis, keep_attrs=keep_attrs, **kwargs)
        return self.apply(reduce_array, shortcut=shortcut)

ops.inject_reduce_methods(DataArrayGroupBy)
ops.inject_binary_ops(DataArrayGroupBy)

class DatasetGroupBy(GroupBy, ImplementsDatasetReduce):
    def apply(self, func, **kwargs):
        """Apply a function over each Dataset in the group and concatenate them
        together into a new Dataset.

        `func` is called like `func(ds, *args, **kwargs)` for each dataset `ds`
        in this group.

        Apply uses heuristics (like `pandas.GroupBy.apply`) to figure out how
        to stack together the datasets. The rule is:
        1. If the dimension along which the group coordinate is defined is
           still in the first grouped item after applying `func`, then stack
           over this dimension.
        2. Otherwise, stack over the new dimension given by name of this
           grouping (the argument to the `groupby` function).

        Parameters
        ----------
        func : function
            Callable to apply to each sub-dataset.
        **kwargs
            Used to call `func(ds, **kwargs)` for each sub-dataset `ar`.

        Returns
        -------
        applied : Dataset
            The result of splitting, applying and combining this dataset.
        """
        kwargs.pop('shortcut', None)  # ignore shortcut if set (for now)
        applied = (func(ds, **kwargs) for ds in self._iter_grouped())
        combined = self._concat(applied)
        result = self._maybe_restore_empty_groups(combined)
        return result

    def _concat(self, applied):
        applied_example, applied = peek_at(applied)
        concat_dim, positions = self._infer_concat_args(applied_example)
        combined = concat(applied, concat_dim, positions=positions)
        return combined

    def reduce(self, func, dim=None, keep_attrs=False, **kwargs):
        """Reduce the items in this group by applying `func` along some
        dimension(s).

        Parameters
        ----------
        func : function
            Function which can be called in the form
            `func(x, axis=axis, **kwargs)` to return the result of collapsing an
            np.ndarray over an integer valued axis.
        dim : str or sequence of str, optional
            Dimension(s) over which to apply `func`.
        axis : int or sequence of int, optional
            Axis(es) over which to apply `func`. Only one of the 'dimension'
            and 'axis' arguments can be supplied. If neither are supplied, then
            `func` is calculated over all dimension for each group item.
        keep_attrs : bool, optional
            If True, the datasets's attributes (`attrs`) will be copied from
            the original object to the new one.  If False (default), the new
            object will be returned without attributes.
        **kwargs : dict
            Additional keyword arguments passed on to `func`.

        Returns
        -------
        reduced : Array
            Array with summarized data and the indicated dimension(s)
            removed.
        """
        def reduce_dataset(ds):
            return ds.reduce(func, dim, keep_attrs, **kwargs)
        return self.apply(reduce_dataset)

    def assign(self, **kwargs):
        """Assign data variables by group.

        See also
        --------
        Dataset.assign
        """
        return self.apply(lambda ds: ds.assign(**kwargs))

ops.inject_reduce_methods(DatasetGroupBy)
ops.inject_binary_ops(DatasetGroupBy)


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 30 16:43:00 2014

@author: jahad
"""

import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
import os
def phasePlot(fp,fm,seqname,saveAs):
    if(os.path.exists(saveAs)):
        os.remove(saveAs)
    for x,y,label in zip(fp,fm,seqname):
        plt.scatter(x,y,marker='.',color='Black')
        plt.annotate(label,xy=(x+.01,y+.01))
    reg1, = plt.fill([0,0,.25],[0,.25,0],color = 'Chartreuse',alpha=.75)
    reg2, = plt.fill([0,0,.35,.25],[.25,.35,0,0],color = 'MediumSeaGreen',alpha=.75)
    reg3, = plt.fill([0,.35,.65,.35],[.35,.65,.35,0],color = 'DarkGreen',alpha=.75)
    reg4, = plt.fill([0,0,.35],[.35,1,.65],color = 'Red',alpha=.75)
    reg5, = plt.fill([.35,.65,1],[0,.35,0],color = 'Blue',alpha=.75)
    plt.ylim([0,1])
    plt.xlim([0,1])
    plt.xlabel('f+')
    plt.ylabel('f-')
    plt.title('Phase Diagram')
    fontP = FontProperties()
    fontP.set_size('x-small')
    plt.legend([reg1,reg2,reg3,reg4,reg5],
               ['Weak Polyampholytes & Polyelectrolytes:\nGlobules & Tadpoles',
                                           'Boundary Region',
                                           'Strong Polyampholytes:\nCoils, Hairpins, Chimeras',
                                           'Negatively Charged Strong Polyelectrolytes:\nSwollen Coils',
                                           'Positively Charged Strong Polyelectrolytes:\nSwollen Coils'],
                prop = fontP)
    plt.savefig(saveAs,dpi=200)
    plt.close()
    return plt

def testPhasePlot():
    graph = phasePlot([.65,.32,.15],[.34,.21,.42],['derp1','harro','nyan'],'C:\\Users\\James Ahad\\Documents\\GitHub\\idpserver\\mysite\\output\\test.png')

def testPhasePlotNull():
    graph = phasePlot([],[],[],'/work/jahad/IDP_patterning/idpserver/mysite/output/test.png')

import computation as comp
def NCPRPlot(sequence, bloblen, saveAs):
    if(not sequence is None):
        data = sequence.NCPRdist(bloblen)
        plt.plot(data[0,:], data[1,:])
    else:
        plt.plot([],[])
        plt.xlim([0,50])
    plt.title('NCPR Distribution')
    plt.xlabel('Blob Index')
    plt.ylabel('NCPR')
    plt.ylim([-1.1,1.1])
    plt.savefig(saveAs, dpi=200)
    plt.close()
    return plt

def testNCPRPlot():
    graph = NCPRPlot(comp.Sequence('EEEEEEKKKKEKEKEKEKEKEEEEEEEKKKKKKEKEKEKEKEKEKEKGGGGGGKEKEKE'),5, 'C:\\Users\\James Ahad\\Documents\\GitHub\\idpserver\\mysite\\output\\testNCPR.png')

def SigmaPlot(sequence, bloblen, saveAs):
    if(not sequence is None):
        data = sequence.Sigmadist(bloblen)
        plt.plot(data[0,:], data[1,:])
    else:
        plt.plot([],[])
        plt.xlim([0,50])
    plt.title('Sigma Distribution')
    plt.xlabel('Blob Index')
    plt.ylabel('Sigma')
    plt.ylim([-.1,1.1])
    plt.savefig(saveAs, dpi=200)
    plt.close()
    return plt

def testSigmaPlot():
    graph = SigmaPlot(comp.Sequence('EEEEEEKKKKEKEKEKEKEKEEEEEEEKKKKKKEKEKEKEKEKEKEKGGGGGGKEKEKE'),5, 'C:\\Users\\James Ahad\\Documents\\GitHub\\idpserver\\mysite\\output\\testSigma.png')

def HydroPlot(sequence, bloblen, saveAs):
    if(not sequence is None):
        data = sequence.Hydrodist(bloblen)
        plt.plot(data[0,:], data[1,:])
    else:
        plt.plot([],[])
        plt.xlim([0,50])
    plt.title('Hydropathy Distribution')
    plt.xlabel('Blob Index')
    plt.ylabel('Hydropathy')
    plt.savefig(saveAs, dpi=200)
    plt.close()
    return plt

def testHydroPlot():
    graph = HydroPlot(comp.Sequence('EEEEEEKKKKEKEKEKEKEKEEEEEEEKKKKKKEKEKEKEKEKEKEKGGGGGGKEKEKE'),5, 'C:\\Users\\James Ahad\\Documents\\GitHub\\idpserver\\mysite\\output\\testHydro.png')

testNCPRPlot()
testSigmaPlot()
testHydroPlot()


"""
==================
Two-class AdaBoost
==================

This example fits an AdaBoosted decision stump on a non-linearly separable
classification dataset composed of two "Gaussian quantiles" clusters
(see :func:`sklearn.datasets.make_gaussian_quantiles`) and plots the decision
boundary and decision scores. The distributions of decision scores are shown
separately for samples of class A and B. The predicted class label for each
sample is determined by the sign of the decision score. Samples with decision
scores greater than zero are classified as B, and are otherwise classified
as A. The magnitude of a decision score determines the degree of likeness with
the predicted class label. Additionally, a new dataset could be constructed
containing a desired purity of class B, for example, by only selecting samples
with a decision score above some value.

"""
print(__doc__)

# Author: Noel Dawe <noel.dawe@gmail.com>
#
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_gaussian_quantiles

# Construct dataset
X1, y1 = make_gaussian_quantiles(cov=2.,
                                 n_samples=200, n_features=2,
                                 n_classes=2, random_state=1)
X2, y2 = make_gaussian_quantiles(mean=(3, 3), cov=1.5,
                                 n_samples=300, n_features=2,
                                 n_classes=2, random_state=1)
X = np.concatenate((X1, X2))
y = np.concatenate((y1, - y2 + 1))

# Create and fit an AdaBoosted decision tree
bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1),
                         algorithm="SAMME",
                         n_estimators=200)

bdt.fit(X, y)

plot_colors = "br"
plot_step = 0.02
class_names = "AB"

plt.figure(figsize=(10, 5))

# Plot the decision boundaries
plt.subplot(121)
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                     np.arange(y_min, y_max, plot_step))

Z = bdt.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
cs = plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
plt.axis("tight")

# Plot the training points
for i, n, c in zip(range(2), class_names, plot_colors):
    idx = np.where(y == i)
    plt.scatter(X[idx, 0], X[idx, 1],
                c=c, cmap=plt.cm.Paired,
                label="Class %s" % n)
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.legend(loc='upper right')
plt.xlabel('x')
plt.ylabel('y')
plt.title('Decision Boundary')

# Plot the two-class decision scores
twoclass_output = bdt.decision_function(X)
plot_range = (twoclass_output.min(), twoclass_output.max())
plt.subplot(122)
for i, n, c in zip(range(2), class_names, plot_colors):
    plt.hist(twoclass_output[y == i],
             bins=10,
             range=plot_range,
             facecolor=c,
             label='Class %s' % n,
             alpha=.5)
x1, x2, y1, y2 = plt.axis()
plt.axis((x1, x2, y1, y2 * 1.2))
plt.legend(loc='upper right')
plt.ylabel('Samples')
plt.xlabel('Score')
plt.title('Decision Scores')

plt.tight_layout()
plt.subplots_adjust(wspace=0.35)
plt.show()


# -*- coding: utf-8 -*-
"""
This is an example for creating simple plots from various Neo structures.
It includes a function that generates toy data.
"""
from __future__ import division  # Use same division in Python 2 and 3

import numpy as np
import quantities as pq
from matplotlib import pyplot as plt

import neo

def generate_block(n_segments=3, n_channels=8, n_units=3,
                   data_samples=1000, feature_samples=100):
    """
    Generate a block with a single recording channel group and a number of
    segments, recording channels and units with associated analog signals
    and spike trains.
    """
    feature_len = feature_samples / data_samples

    # Create container and grouping objects
    segments = [neo.Segment(index=i) for i in range(n_segments)]

    rcg = neo.RecordingChannelGroup(name='T0')
    for i in range(n_channels):
        rc = neo.RecordingChannel(name='C%d' % i, index=i)
        rc.recordingchannelgroups = [rcg]
        rcg.recordingchannels.append(rc)

    units = [neo.Unit('U%d' % i) for i in range(n_units)]
    rcg.units = units

    block = neo.Block()
    block.segments = segments
    block.recordingchannelgroups = [rcg]

    # Create synthetic data
    for seg in segments:
        feature_pos = np.random.randint(0, data_samples - feature_samples)

        # Analog signals: Noise with a single sinewave feature
        wave = 3 * np.sin(np.linspace(0, 2 * np.pi, feature_samples))
        for rc in rcg.recordingchannels:
            sig = np.random.randn(data_samples)
            sig[feature_pos:feature_pos + feature_samples] += wave

            signal = neo.AnalogSignal(sig * pq.mV, sampling_rate=1 * pq.kHz)
            seg.analogsignals.append(signal)
            rc.analogsignals.append(signal)

        # Spike trains: Random spike times with elevated rate in short period
        feature_time = feature_pos / data_samples
        for u in units:
            random_spikes = np.random.rand(20)
            feature_spikes = np.random.rand(5) * feature_len + feature_time
            spikes = np.hstack([random_spikes, feature_spikes])

            train = neo.SpikeTrain(spikes * pq.s, 1 * pq.s)
            seg.spiketrains.append(train)
            u.spiketrains.append(train)

    block.create_many_to_one_relationship()
    return block

block = generate_block()

# In this example, we treat each segment in turn, averaging over the channels
# in each:
for seg in block.segments:
    print("Analysing segment %d" % seg.index)

    siglist = seg.analogsignals
    time_points = siglist[0].times
    avg = np.mean(siglist, axis=0)  # Average over signals of Segment

    plt.figure()
    plt.plot(time_points, avg)
    plt.title("Peak response in segment %d: %f" % (seg.index, avg.max()))

# The second alternative is spatial traversal of the data (by channel), with
# averaging over trials. For example, perhaps you wish to see which physical
# location produces the strongest response, and each stimulus was the same:

# We assume that our block has only 1 RecordingChannelGroup and each
# RecordingChannel only has 1 AnalogSignal.
rcg = block.recordingchannelgroups[0]
for rc in rcg.recordingchannels:
    print("Analysing channel %d: %s" % (rc.index, rc.name))

    siglist = rc.analogsignals
    time_points = siglist[0].times
    avg = np.mean(siglist, axis=0)  # Average over signals of RecordingChannel

    plt.figure()
    plt.plot(time_points, avg)
    plt.title("Average response on channel %d" % rc.index)

# There are three ways to access the spike train data: by Segment,
# by RecordingChannel or by Unit.

# By Segment. In this example, each Segment represents data from one trial,
# and we want a peristimulus time histogram (PSTH) for each trial from all
# Units combined:
for seg in block.segments:
    print("Analysing segment %d" % seg.index)
    stlist = [st - st.t_start for st in seg.spiketrains]
    count, bins = np.histogram(np.hstack(stlist))
    plt.figure()
    plt.bar(bins[:-1], count, width=bins[1] - bins[0])
    plt.title("PSTH in segment %d" % seg.index)

# By Unit. Now we can calculate the PSTH averaged over trials for each Unit:
for unit in block.list_units:
    stlist = [st - st.t_start for st in unit.spiketrains]
    count, bins = np.histogram(np.hstack(stlist))
    plt.figure()
    plt.bar(bins[:-1], count, width=bins[1] - bins[0])
    plt.title("PSTH of unit %s" % unit.name)

# By RecordingChannelGroup. Here we calculate a PSTH averaged over trials by
# channel location, blending all Units:
for rcg in block.recordingchannelgroups:
    stlist = []
    for unit in rcg.units:
        stlist.extend([st - st.t_start for st in unit.spiketrains])
    count, bins = np.histogram(np.hstack(stlist))
    plt.figure()
    plt.bar(bins[:-1], count, width=bins[1] - bins[0])
    plt.title("PSTH blend of recording channel group %s" % rcg.name)

plt.show()


from typing import Any

import pandas as pd
import pandas.util.testing as tm
import pytest
from multipledispatch.conflict import ambiguities

import ibis
import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.operations as ops
from ibis.pandas.client import PandasClient
from ibis.pandas.core import is_computable_input
from ibis.pandas.dispatch import execute_node, post_execute, pre_execute

pytestmark = pytest.mark.pandas

@pytest.fixture
def dataframe():
    return pd.DataFrame(
        {
            'plain_int64': list(range(1, 4)),
            'plain_strings': list('abc'),
            'dup_strings': list('dad'),
        }
    )

@pytest.fixture
def core_client(dataframe):
    return ibis.pandas.connect({'df': dataframe})

@pytest.fixture
def ibis_table(core_client):
    return core_client.table('df')

@pytest.mark.parametrize('func', [execute_node, pre_execute, post_execute])
def test_no_execute_ambiguities(func):
    assert not ambiguities(func.funcs)

def test_from_dataframe(dataframe, ibis_table, core_client):
    t = ibis.pandas.from_dataframe(dataframe)
    result = t.execute()
    expected = ibis_table.execute()
    tm.assert_frame_equal(result, expected)

    t = ibis.pandas.from_dataframe(dataframe, name='foo')
    expected = ibis_table.execute()
    tm.assert_frame_equal(result, expected)

    client = core_client
    t = ibis.pandas.from_dataframe(dataframe, name='foo', client=client)
    expected = ibis_table.execute()
    tm.assert_frame_equal(result, expected)

def test_pre_execute_basic():
    """
    Test that pre_execute has intercepted execution and provided its own
    scope dict
    """

    @pre_execute.register(ops.Add)
    def pre_execute_test(op, *clients, scope=None, **kwargs):
        return {op: 4}

    one = ibis.literal(1)
    expr = one + one
    result = ibis.pandas.execute(expr)
    assert result == 4

    del pre_execute.funcs[(ops.Add,)]
    pre_execute.reorder()
    pre_execute._cache.clear()

def test_execute_parameter_only():
    param = ibis.param('int64')
    result = ibis.pandas.execute(param, params={param: 42})
    assert result == 42

def test_missing_data_sources():
    t = ibis.table([('a', 'string')])
    expr = t.a.length()
    with pytest.raises(com.UnboundExpressionError):
        ibis.pandas.execute(expr)

def test_missing_data_on_custom_client():
    class MyClient(PandasClient):
        def table(self, name):
            return ops.DatabaseTable(
                name, ibis.schema([('a', 'int64')]), self
            ).to_expr()

    con = MyClient({})
    t = con.table('t')
    with pytest.raises(
        NotImplementedError,
        match=(
            'Could not find signature for execute_node: '
            '<DatabaseTable, MyClient>'
        ),
    ):
        con.execute(t)

def test_post_execute_called_on_joins(dataframe, core_client, ibis_table):
    count = [0]

    @post_execute.register(ops.InnerJoin, pd.DataFrame)
    def tmp_left_join_exe(op, lhs, **kwargs):
        count[0] += 1
        return lhs

    left = ibis_table
    right = left.view()
    join = left.join(right, 'plain_strings')[left.plain_int64]
    result = join.execute()
    assert result is not None
    assert not result.empty
    assert count[0] == 1

def test_is_computable_input():
    class MyObject:
        def __init__(self, value: float) -> None:
            self.value = value

        def __getattr__(self, name: str) -> Any:
            return getattr(self.value, name)

        def __hash__(self) -> int:
            return hash((type(self), self.value))

        def __eq__(self, other):
            return (
                isinstance(other, type(self))
                and isinstance(self, type(other))
                and self.value == other.value
            )

        def __float__(self) -> float:
            return self.value

    @execute_node.register(ops.Add, int, MyObject)
    def add_int_my_object(op, left, right, **kwargs):
        return left + right.value

    # This multimethod must be implemented to play nicely with other value
    # types like columns and literals. In other words, for a custom
    # non-expression object to play nicely it must somehow map to one of the
    # types in ibis/expr/datatypes.py
    @dt.infer.register(MyObject)
    def infer_my_object(_, **kwargs):
        return dt.float64

    @is_computable_input.register(MyObject)
    def is_computable_input_my_object(_):
        return True

    one = ibis.literal(1)
    two = MyObject(2.0)
    assert is_computable_input(two)

    three = one + two
    four = three + 1
    result = ibis.pandas.execute(four)
    assert result == 4.0

    del execute_node.funcs[ops.Add, int, MyObject]
    execute_node.reorder()
    execute_node._cache.clear()

    del dt.infer.funcs[(MyObject,)]
    dt.infer.reorder()
    dt.infer._cache.clear()


from bs4 import BeautifulSoup
from urllib2 import quote
import unicodedata
import requests
import json
import glob
import pandas as pd

movie_list = []

for page in glob.glob('*.html'):
    with open(page, 'r+') as f:
        my_page = f.read()
        my_soup = BeautifulSoup(my_page)
        for div in my_soup.find_all('div', class_='lockup'):
            try:
                movie_list.append(div.img.get('alt'))
            except:
                movie_list.append('movie could not be extracted from page')
['movie could not be extracted from page' for movie in movie_list if movie is None]

movie_list2 = []

for movie in movie_list:
    try:
        movie = quote(movie)
        movie_list2.append(movie)
    except:
        try:
            movie = unicodedata.normalize('NFKC', movie).encode('ascii','ignore')
            movie = quote(movie)
            movie_list2.append(movie)
        except:
            print movie
            movie_list2.append('movie could not be processed')

all_movies_us = {}
for movie in movie_list2:
    try:
        query_url = 'http://www.omdbapi.com/?t=' + movie + '&y=&plot=full&r=json'
        response = requests.get(query_url)
        my_dict = json.loads(response.text)
        all_movies_us[movie] = my_dict
    except:
        all_movies_us[movie] = 'No response'

    print movie

# movies/single year shows
years_dict = {}
counter = 0
for k,v in all_movies.iteritems():
    try:
        if len(v['Year']) == 4:
            try:
                years_dict[v['Year']] += 1
            except:
                years_dict[v['Year']] = 1
                continue

    except:
        counter += 1
        continue
print counter
my_frame = pd.DataFrame.from_dict(years_dict, orient = 'index')
my_frame.to_csv('single_years.csv')

counter=0
score_dict = {}
for k,v in all_movies.iteritems():
    try:
        if v['imdbRating'] != 'N/A':
            score_dict[v['Title']] = v['imdbRating']
    except:
        counter +=1
        continue
print counter

score_dict2 ={}
for title, score in score_dict.iteritems():
    try:
        score_dict2[title] = float(score)
    except:
        print score
score_dict = score_dict2

average_score = (sum(score_dict.values()))/len(score_dict)
top_25 =

print average_score

years = []
country = []
language  =[]
actors = []

for movie, results in all_movies.iteritems():
    try:
        years.append(results['Year'])
    except:
        continue
    try:
        country.append(results['Country'])
    except:
        continue
    try:
        language.append(results['Language'])
    except:
        continue
    try:
        for actor in results['Actors'].split(','):
            actors.append(actor)
    except:
        continue

# Ongoing shows
years_dict = {}
counter = 0
for k,v in all_movies.iteritems():
    try:
        print v['Year'][4]
    except:
        continue

# Languages
lang_list = []
for lang in language:
    for x in lang.split(','):
        lang_list.append(x)
lang_list
Counter(lang_list)
Counter(lang_list).most_common(10)


import pandas as pd
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import (QWidget, QLabel, QHBoxLayout,
                             QVBoxLayout, QPushButton, QSlider,
                             QComboBox)

from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure

from .FilterCvQtContainer import Filter

import random

class SinglePlotContainer(QWidget):

    num_plots = 0

    def __init__(self, parent=None):
        super().__init__()

        self.num_plots += 1

        self.variable_df = pd.DataFrame()

        self.figure = Figure() # don't use matplotlib.pyplot at all!
        self.canvas = FigureCanvas(self.figure)

        self.hLayout = QHBoxLayout(self)
        self.dataConfigColumn = QVBoxLayout()
        self.filtersColumn = QVBoxLayout()

        self.hLayout.addLayout(self.dataConfigColumn)
        self.hLayout.addWidget(self.canvas)
        self.hLayout.addLayout(self.filtersColumn)

        self.comboLoadVariable = QComboBox()
        self.dataConfigColumn.addWidget(self.comboLoadVariable)

        self.filter1 = Filter('Moving Average', 3, 30, 5, 1)
        self.filtersColumn.addWidget(self.filter1)

        # drawEvent = self.figure.canvas.mpl_connect('draw', self.updatePlot)

        self.plotRandom()

    def connectButtons(self):
        self.comboLoadVariable.activated[str].connect(self.loadVariable)

    def loadVariable(self, variable):
        self.variable_df = self.parent().parent().original_df[variable]
        self.plot()

    def plot(self):
        if self.num_plots != 0:
            self.axes = self.figure.add_subplot(111, sharex=self.parent().parent().plots[0].axes)
        else:
            self.axes = self.figure.add_subplot(111)
        self.axes.clear()
        self.axes.plot(self.variable_df, '-')
        self.canvas.draw()

    def updatePlot(self):
        ymax,ymin = self.axes.get_ylim()
        self.axes.clear()
        self.axes.set_ylim(ymax,ymin)
        self.axes.plot(self.variable_df, '-')
        self.canvas.draw()

    def plotRandom(self):
        ''' plot some random stuff '''
        data = [random.random() for i in range(10)]
        self.axes = self.figure.add_subplot(111)
        self.axes.clear()
        self.axes.plot(data, '-')
        self.canvas.draw()


"""
======================================================
Classification of text documents using sparse features
======================================================

This is an example showing how scikit-learn can be used to classify documents
by topics using a bag-of-words approach. This example uses a scipy.sparse
matrix to store the features and demonstrates various classifiers that can
efficiently handle sparse matrices.

The dataset used in this example is the 20 newsgroups dataset. It will be
automatically downloaded, then cached.

The bar plot indicates the accuracy, training time (normalized) and test time
(normalized) of each classifier.

"""

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Lars Buitinck <L.J.Buitinck@uva.nl>
# License: BSD 3 clause

from __future__ import print_function

import logging
import numpy as np
from optparse import OptionParser
import sys
from time import time
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import RidgeClassifier
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.ensemble import RandomForestClassifier
from sklearn.utils.extmath import density
from sklearn import metrics

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

# parse commandline arguments
op = OptionParser()
op.add_option("--report",
              action="store_true", dest="print_report",
              help="Print a detailed classification report.")
op.add_option("--chi2_select",
              action="store", type="int", dest="select_chi2",
              help="Select some number of features using a chi-squared test")
op.add_option("--confusion_matrix",
              action="store_true", dest="print_cm",
              help="Print the confusion matrix.")
op.add_option("--top10",
              action="store_true", dest="print_top10",
              help="Print ten most discriminative terms per class"
                   " for every classifier.")
op.add_option("--all_categories",
              action="store_true", dest="all_categories",
              help="Whether to use all categories or not.")
op.add_option("--use_hashing",
              action="store_true",
              help="Use a hashing vectorizer.")
op.add_option("--n_features",
              action="store", type=int, default=2 ** 16,
              help="n_features when using the hashing vectorizer.")
op.add_option("--filtered",
              action="store_true",
              help="Remove newsgroup information that is easily overfit: "
                   "headers, signatures, and quoting.")

(opts, args) = op.parse_args()
if len(args) > 0:
    op.error("this script takes no arguments.")
    sys.exit(1)

print(__doc__)
op.print_help()
print()

###############################################################################
# Load some categories from the training set
if opts.all_categories:
    categories = None
else:
    categories = [
        'alt.atheism',
        'talk.religion.misc',
        'comp.graphics',
        'sci.space',
    ]

if opts.filtered:
    remove = ('headers', 'footers', 'quotes')
else:
    remove = ()

print("Loading 20 newsgroups dataset for categories:")
print(categories if categories else "all")

data_train = fetch_20newsgroups(subset='train', categories=categories,
                                shuffle=True, random_state=42,
                                remove=remove)

data_test = fetch_20newsgroups(subset='test', categories=categories,
                               shuffle=True, random_state=42,
                               remove=remove)
print('data loaded')

categories = data_train.target_names    # for case categories == None

def size_mb(docs):
    return sum(len(s.encode('utf-8')) for s in docs) / 1e6

data_train_size_mb = size_mb(data_train.data)
data_test_size_mb = size_mb(data_test.data)

print("%d documents - %0.3fMB (training set)" % (
    len(data_train.data), data_train_size_mb))
print("%d documents - %0.3fMB (test set)" % (
    len(data_test.data), data_test_size_mb))
print("%d categories" % len(categories))
print()

# split a training set and a test set
y_train, y_test = data_train.target, data_test.target

print("Extracting features from the training data using a sparse vectorizer")
t0 = time()
if opts.use_hashing:
    vectorizer = HashingVectorizer(stop_words='english', non_negative=True,
                                   n_features=opts.n_features)
    X_train = vectorizer.transform(data_train.data)
else:
    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
                                 stop_words='english')
    X_train = vectorizer.fit_transform(data_train.data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_train_size_mb / duration))
print("n_samples: %d, n_features: %d" % X_train.shape)
print()

print("Extracting features from the test data using the same vectorizer")
t0 = time()
X_test = vectorizer.transform(data_test.data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_test_size_mb / duration))
print("n_samples: %d, n_features: %d" % X_test.shape)
print()

# mapping from integer feature name to original token string
if opts.use_hashing:
    feature_names = None
else:
    feature_names = vectorizer.get_feature_names()

if opts.select_chi2:
    print("Extracting %d best features by a chi-squared test" %
          opts.select_chi2)
    t0 = time()
    ch2 = SelectKBest(chi2, k=opts.select_chi2)
    X_train = ch2.fit_transform(X_train, y_train)
    X_test = ch2.transform(X_test)
    if feature_names:
        # keep selected feature names
        feature_names = [feature_names[i] for i
                         in ch2.get_support(indices=True)]
    print("done in %fs" % (time() - t0))
    print()

if feature_names:
    feature_names = np.asarray(feature_names)

def trim(s):
    """Trim string to fit on terminal (assuming 80-column display)"""
    return s if len(s) <= 80 else s[:77] + "..."

###############################################################################
# Benchmark classifiers
def benchmark(clf):
    print('_' * 80)
    print("Training: ")
    print(clf)
    t0 = time()
    clf.fit(X_train, y_train)
    train_time = time() - t0
    print("train time: %0.3fs" % train_time)

    t0 = time()
    pred = clf.predict(X_test)
    test_time = time() - t0
    print("test time:  %0.3fs" % test_time)

    score = metrics.accuracy_score(y_test, pred)
    print("accuracy:   %0.3f" % score)

    if hasattr(clf, 'coef_'):
        print("dimensionality: %d" % clf.coef_.shape[1])
        print("density: %f" % density(clf.coef_))

        if opts.print_top10 and feature_names is not None:
            print("top 10 keywords per class:")
            for i, category in enumerate(categories):
                top10 = np.argsort(clf.coef_[i])[-10:]
                print(trim("%s: %s"
                      % (category, " ".join(feature_names[top10]))))
        print()

    if opts.print_report:
        print("classification report:")
        print(metrics.classification_report(y_test, pred,
                                            target_names=categories))

    if opts.print_cm:
        print("confusion matrix:")
        print(metrics.confusion_matrix(y_test, pred))

    print()
    clf_descr = str(clf).split('(')[0]
    return clf_descr, score, train_time, test_time

results = []
for clf, name in (
        (RidgeClassifier(tol=1e-2, solver="lsqr"), "Ridge Classifier"),
        (Perceptron(n_iter=50), "Perceptron"),
        (PassiveAggressiveClassifier(n_iter=50), "Passive-Aggressive"),
        (KNeighborsClassifier(n_neighbors=10), "kNN"),
        (RandomForestClassifier(n_estimators=100), "Random forest")):
    print('=' * 80)
    print(name)
    results.append(benchmark(clf))

for penalty in ["l2", "l1"]:
    print('=' * 80)
    print("%s penalty" % penalty.upper())
    # Train Liblinear model
    results.append(benchmark(LinearSVC(loss='l2', penalty=penalty,
                                            dual=False, tol=1e-3)))

    # Train SGD model
    results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                           penalty=penalty)))

# Train SGD with Elastic Net penalty
print('=' * 80)
print("Elastic-Net penalty")
results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                       penalty="elasticnet")))

# Train NearestCentroid without threshold
print('=' * 80)
print("NearestCentroid (aka Rocchio classifier)")
results.append(benchmark(NearestCentroid()))

# Train sparse Naive Bayes classifiers
print('=' * 80)
print("Naive Bayes")
results.append(benchmark(MultinomialNB(alpha=.01)))
results.append(benchmark(BernoulliNB(alpha=.01)))

print('=' * 80)
print("LinearSVC with L1-based feature selection")
# The smaller C, the stronger the regularization.
# The more regularization, the more sparsity.
results.append(benchmark(Pipeline([
  ('feature_selection', LinearSVC(penalty="l1", dual=False, tol=1e-3)),
  ('classification', LinearSVC())
])))

# make some plots

indices = np.arange(len(results))

results = [[x[i] for x in results] for i in range(4)]

clf_names, score, training_time, test_time = results
training_time = np.array(training_time) / np.max(training_time)
test_time = np.array(test_time) / np.max(test_time)

plt.figure(figsize=(12, 8))
plt.title("Score")
plt.barh(indices, score, .2, label="score", color='r')
plt.barh(indices + .3, training_time, .2, label="training time", color='g')
plt.barh(indices + .6, test_time, .2, label="test time", color='b')
plt.yticks(())
plt.legend(loc='best')
plt.subplots_adjust(left=.25)
plt.subplots_adjust(top=.95)
plt.subplots_adjust(bottom=.05)

for i, c in zip(indices, clf_names):
    plt.text(-.3, i, c)

plt.show()


"""Benchmarks of orthogonal matching pursuit (:ref:`OMP`) versus least angle
regression (:ref:`least_angle_regression`)

The input data is mostly low rank but is a fat infinite tail.
"""
from __future__ import print_function

import gc
import sys
from time import time

import numpy as np

from sklearn.linear_model import lars_path, orthogonal_mp
from sklearn.datasets.samples_generator import make_sparse_coded_signal

def compute_bench(samples_range, features_range):

    it = 0

    results = dict()
    lars = np.empty((len(features_range), len(samples_range)))
    lars_gram = lars.copy()
    omp = lars.copy()
    omp_gram = lars.copy()

    max_it = len(samples_range) * len(features_range)
    for i_s, n_samples in enumerate(samples_range):
        for i_f, n_features in enumerate(features_range):
            it += 1
            n_informative = n_features / 10
            print('====================')
            print('Iteration %03d of %03d' % (it, max_it))
            print('====================')
            # dataset_kwargs = {
            #     'n_train_samples': n_samples,
            #     'n_test_samples': 2,
            #     'n_features': n_features,
            #     'n_informative': n_informative,
            #     'effective_rank': min(n_samples, n_features) / 10,
            #     #'effective_rank': None,
            #     'bias': 0.0,
            # }
            dataset_kwargs = {
                'n_samples': 1,
                'n_components': n_features,
                'n_features': n_samples,
                'n_nonzero_coefs': n_informative,
                'random_state': 0
            }
            print("n_samples: %d" % n_samples)
            print("n_features: %d" % n_features)
            y, X, _ = make_sparse_coded_signal(**dataset_kwargs)
            X = np.asfortranarray(X)

            gc.collect()
            print("benchmarking lars_path (with Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            G = np.dot(X.T, X)  # precomputed Gram matrix
            Xy = np.dot(X.T, y)
            lars_path(X, y, Xy=Xy, Gram=G, max_iter=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            lars_gram[i_f, i_s] = delta

            gc.collect()
            print("benchmarking lars_path (without Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            lars_path(X, y, Gram=None, max_iter=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            lars[i_f, i_s] = delta

            gc.collect()
            print("benchmarking orthogonal_mp (with Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            orthogonal_mp(X, y, precompute=True,
                          n_nonzero_coefs=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            omp_gram[i_f, i_s] = delta

            gc.collect()
            print("benchmarking orthogonal_mp (without Gram):", end='')
            sys.stdout.flush()
            tstart = time()
            orthogonal_mp(X, y, precompute=False,
                          n_nonzero_coefs=n_informative)
            delta = time() - tstart
            print("%0.3fs" % delta)
            omp[i_f, i_s] = delta

    results['time(LARS) / time(OMP)\n (w/ Gram)'] = (lars_gram / omp_gram)
    results['time(LARS) / time(OMP)\n (w/o Gram)'] = (lars / omp)
    return results

if __name__ == '__main__':
    samples_range = np.linspace(1000, 5000, 5).astype(np.int)
    features_range = np.linspace(1000, 5000, 5).astype(np.int)
    results = compute_bench(samples_range, features_range)
    max_time = max(np.max(t) for t in results.values())

    import matplotlib.pyplot as plt
    fig = plt.figure('scikit-learn OMP vs. LARS benchmark results')
    for i, (label, timings) in enumerate(sorted(results.iteritems())):
        ax = fig.add_subplot(1, 2, i+1)
        vmax = max(1 - timings.min(), -1 + timings.max())
        plt.matshow(timings, fignum=False, vmin=1 - vmax, vmax=1 + vmax)
        ax.set_xticklabels([''] + map(str, samples_range))
        ax.set_yticklabels([''] + map(str, features_range))
        plt.xlabel('n_samples')
        plt.ylabel('n_features')
        plt.title(label)

    plt.subplots_adjust(0.1, 0.08, 0.96, 0.98, 0.4, 0.63)
    ax = plt.axes([0.1, 0.08, 0.8, 0.06])
    plt.colorbar(cax=ax, orientation='horizontal')
    plt.show()


import numpy as np
from scipy import sparse

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_false

from sklearn.preprocessing.imputation import Imputer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn import tree
from sklearn.random_projection import sparse_random_matrix

def _check_statistics(X, X_true,
                      strategy, statistics, missing_values):
    """Utility function for testing imputation for a given strategy.

    Test:
        - along the two axes
        - with dense and sparse arrays

    Check that:
        - the statistics (mean, median, mode) are correct
        - the missing values are imputed correctly"""

    err_msg = "Parameters: strategy = %s, missing_values = %s, " \
              "axis = {0}, sparse = {1}" % (strategy, missing_values)

    # Normal matrix, axis = 0
    imputer = Imputer(missing_values, strategy=strategy, axis=0)
    X_trans = imputer.fit(X).transform(X.copy())
    assert_array_equal(imputer.statistics_, statistics,
                       err_msg.format(0, False))
    assert_array_equal(X_trans, X_true, err_msg.format(0, False))

    # Normal matrix, axis = 1
    imputer = Imputer(missing_values, strategy=strategy, axis=1)
    imputer.fit(X.transpose())
    if np.isnan(statistics).any():
        assert_raises(ValueError, imputer.transform, X.copy().transpose())
    else:
        X_trans = imputer.transform(X.copy().transpose())
        assert_array_equal(X_trans, X_true.transpose(),
                           err_msg.format(1, False))

    # Sparse matrix, axis = 0
    imputer = Imputer(missing_values, strategy=strategy, axis=0)
    imputer.fit(sparse.csc_matrix(X))
    X_trans = imputer.transform(sparse.csc_matrix(X.copy()))

    if sparse.issparse(X_trans):
        X_trans = X_trans.toarray()

    assert_array_equal(imputer.statistics_, statistics,
                       err_msg.format(0, True))
    assert_array_equal(X_trans, X_true, err_msg.format(0, True))

    # Sparse matrix, axis = 1
    imputer = Imputer(missing_values, strategy=strategy, axis=1)
    imputer.fit(sparse.csc_matrix(X.transpose()))
    if np.isnan(statistics).any():
        assert_raises(ValueError, imputer.transform,
                      sparse.csc_matrix(X.copy().transpose()))
    else:
        X_trans = imputer.transform(sparse.csc_matrix(X.copy().transpose()))

        if sparse.issparse(X_trans):
            X_trans = X_trans.toarray()

        assert_array_equal(X_trans, X_true.transpose(),
                           err_msg.format(1, True))

def test_imputation_shape():
    # Verify the shapes of the imputed matrix for different strategies.
    X = np.random.randn(10, 2)
    X[::2] = np.nan

    for strategy in ['mean', 'median', 'most_frequent']:
        imputer = Imputer(strategy=strategy)
        X_imputed = imputer.fit_transform(X)
        assert_equal(X_imputed.shape, (10, 2))
        X_imputed = imputer.fit_transform(sparse.csr_matrix(X))
        assert_equal(X_imputed.shape, (10, 2))

def test_imputation_mean_median_only_zero():
    # Test imputation using the mean and median strategies, when
    # missing_values == 0.
    X = np.array([
        [np.nan, 0, 0, 0, 5],
        [np.nan, 1, 0, np.nan, 3],
        [np.nan, 2, 0, 0, 0],
        [np.nan, 6, 0, 5, 13],
    ])

    X_imputed_mean = np.array([
        [3, 5],
        [1, 3],
        [2, 7],
        [6, 13],
    ])
    statistics_mean = [np.nan, 3, np.nan, np.nan, 7]

    # Behaviour of median with NaN is undefined, e.g. different results in
    # np.median and np.ma.median
    X_for_median = X[:, [0, 1, 2, 4]]
    X_imputed_median = np.array([
        [2, 5],
        [1, 3],
        [2, 5],
        [6, 13],
    ])
    statistics_median = [np.nan, 2, np.nan, 5]

    _check_statistics(X, X_imputed_mean, "mean", statistics_mean, 0)
    _check_statistics(X_for_median, X_imputed_median, "median",
                      statistics_median, 0)

def safe_median(arr, *args, **kwargs):
    # np.median([]) raises a TypeError for numpy >= 1.10.1
    length = arr.size if hasattr(arr, 'size') else len(arr)
    return np.nan if length == 0 else np.median(arr, *args, **kwargs)

def safe_mean(arr, *args, **kwargs):
    # np.mean([]) raises a RuntimeWarning for numpy >= 1.10.1
    length = arr.size if hasattr(arr, 'size') else len(arr)
    return np.nan if length == 0 else np.mean(arr, *args, **kwargs)

def test_imputation_mean_median():
    # Test imputation using the mean and median strategies, when
    # missing_values != 0.
    rng = np.random.RandomState(0)

    dim = 10
    dec = 10
    shape = (dim * dim, dim + dec)

    zeros = np.zeros(shape[0])
    values = np.arange(1, shape[0] + 1)
    values[4::2] = - values[4::2]

    tests = [("mean", "NaN", lambda z, v, p: safe_mean(np.hstack((z, v)))),
             ("mean", 0, lambda z, v, p: np.mean(v)),
             ("median", "NaN", lambda z, v, p: safe_median(np.hstack((z, v)))),
             ("median", 0, lambda z, v, p: np.median(v))]

    for strategy, test_missing_values, true_value_fun in tests:
        X = np.empty(shape)
        X_true = np.empty(shape)
        true_statistics = np.empty(shape[1])

        # Create a matrix X with columns
        #    - with only zeros,
        #    - with only missing values
        #    - with zeros, missing values and values
        # And a matrix X_true containing all true values
        for j in range(shape[1]):
            nb_zeros = (j - dec + 1 > 0) * (j - dec + 1) * (j - dec + 1)
            nb_missing_values = max(shape[0] + dec * dec
                                    - (j + dec) * (j + dec), 0)
            nb_values = shape[0] - nb_zeros - nb_missing_values

            z = zeros[:nb_zeros]
            p = np.repeat(test_missing_values, nb_missing_values)
            v = values[rng.permutation(len(values))[:nb_values]]

            true_statistics[j] = true_value_fun(z, v, p)

            # Create the columns
            X[:, j] = np.hstack((v, z, p))

            if 0 == test_missing_values:
                X_true[:, j] = np.hstack((v,
                                          np.repeat(
                                              true_statistics[j],
                                              nb_missing_values + nb_zeros)))
            else:
                X_true[:, j] = np.hstack((v,
                                          z,
                                          np.repeat(true_statistics[j],
                                                    nb_missing_values)))

            # Shuffle them the same way
            np.random.RandomState(j).shuffle(X[:, j])
            np.random.RandomState(j).shuffle(X_true[:, j])

        # Mean doesn't support columns containing NaNs, median does
        if strategy == "median":
            cols_to_keep = ~np.isnan(X_true).any(axis=0)
        else:
            cols_to_keep = ~np.isnan(X_true).all(axis=0)

        X_true = X_true[:, cols_to_keep]

        _check_statistics(X, X_true, strategy,
                          true_statistics, test_missing_values)

def test_imputation_median_special_cases():
    # Test median imputation with sparse boundary cases
    X = np.array([
        [0, np.nan, np.nan],  # odd: implicit zero
        [5, np.nan, np.nan],  # odd: explicit nonzero
        [0, 0, np.nan],    # even: average two zeros
        [-5, 0, np.nan],   # even: avg zero and neg
        [0, 5, np.nan],    # even: avg zero and pos
        [4, 5, np.nan],    # even: avg nonzeros
        [-4, -5, np.nan],  # even: avg negatives
        [-1, 2, np.nan],   # even: crossing neg and pos
    ]).transpose()

    X_imputed_median = np.array([
        [0, 0, 0],
        [5, 5, 5],
        [0, 0, 0],
        [-5, 0, -2.5],
        [0, 5, 2.5],
        [4, 5, 4.5],
        [-4, -5, -4.5],
        [-1, 2, .5],
    ]).transpose()
    statistics_median = [0, 5, 0, -2.5, 2.5, 4.5, -4.5, .5]

    _check_statistics(X, X_imputed_median, "median",
                      statistics_median, 'NaN')

def test_imputation_most_frequent():
    # Test imputation using the most-frequent strategy.
    X = np.array([
        [-1, -1, 0, 5],
        [-1, 2, -1, 3],
        [-1, 1, 3, -1],
        [-1, 2, 3, 7],
    ])

    X_true = np.array([
        [2, 0, 5],
        [2, 3, 3],
        [1, 3, 3],
        [2, 3, 7],
    ])

    # scipy.stats.mode, used in Imputer, doesn't return the first most
    # frequent as promised in the doc but the lowest most frequent. When this
    # test will fail after an update of scipy, Imputer will need to be updated
    # to be consistent with the new (correct) behaviour
    _check_statistics(X, X_true, "most_frequent", [np.nan, 2, 3, 3], -1)

def test_imputation_pipeline_grid_search():
    # Test imputation within a pipeline + gridsearch.
    pipeline = Pipeline([('imputer', Imputer(missing_values=0)),
                         ('tree', tree.DecisionTreeRegressor(random_state=0))])

    parameters = {
        'imputer__strategy': ["mean", "median", "most_frequent"],
        'imputer__axis': [0, 1]
    }

    l = 100
    X = sparse_random_matrix(l, l, density=0.10)
    Y = sparse_random_matrix(l, 1, density=0.10).toarray()
    gs = GridSearchCV(pipeline, parameters)
    gs.fit(X, Y)

def test_imputation_pickle():
    # Test for pickling imputers.
    import pickle

    l = 100
    X = sparse_random_matrix(l, l, density=0.10)

    for strategy in ["mean", "median", "most_frequent"]:
        imputer = Imputer(missing_values=0, strategy=strategy)
        imputer.fit(X)

        imputer_pickled = pickle.loads(pickle.dumps(imputer))

        assert_array_equal(imputer.transform(X.copy()),
                           imputer_pickled.transform(X.copy()),
                           "Fail to transform the data after pickling "
                           "(strategy = %s)" % (strategy))

def test_imputation_copy():
    # Test imputation with copy
    X_orig = sparse_random_matrix(5, 5, density=0.75, random_state=0)

    # copy=True, dense => copy
    X = X_orig.copy().toarray()
    imputer = Imputer(missing_values=0, strategy="mean", copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert_false(np.all(X == Xt))

    # copy=True, sparse csr => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean", copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_false(np.all(X.data == Xt.data))

    # copy=False, dense => no copy
    X = X_orig.copy().toarray()
    imputer = Imputer(missing_values=0, strategy="mean", copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert_array_equal(X, Xt)

    # copy=False, sparse csr, axis=1 => no copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=1)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_array_equal(X.data, Xt.data)

    # copy=False, sparse csc, axis=0 => no copy
    X = X_orig.copy().tocsc()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=0)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_array_equal(X.data, Xt.data)

    # copy=False, sparse csr, axis=0 => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=0)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_false(np.all(X.data == Xt.data))

    # copy=False, sparse csc, axis=1 => copy
    X = X_orig.copy().tocsc()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                      copy=False, axis=1)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_false(np.all(X.data == Xt.data))

    # copy=False, sparse csr, axis=1, missing_values=0 => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=0, strategy="mean",
                      copy=False, axis=1)
    Xt = imputer.fit(X).transform(X)
    assert_false(sparse.issparse(Xt))

    # Note: If X is sparse and if missing_values=0, then a (dense) copy of X is
    # made, even if copy=False.


# Simply uses the assessors estimate to predict price, so we can see how much better the machine learning models are.
# requires data from Assemble_Data.py

# Copyright (C) 2017  Kevin Maher

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Data for this project may be the property of the Boulder County Assessor's office,
# they gave me free access as a student but were not clear about any restrictions regarding
# sharing the URL from which the data was downloaded.
# The data has been pre-processed from xlsx to csv files because OpenOffice had
# problems with the xlsx files.
# Data was pre-processed by a data setup script, Assemble_Data.py which produced the
# file '$working_data_5c.csv'

import pandas as pd
import numpy as np
from math import sqrt
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor
from sklearn.ensemble import GradientBoostingRegressor, AdaBoostRegressor
from sklearn.linear_model import LinearRegression

# https://stats.stackexchange.com/questions/58391/mean-absolute-percentage-error-mape-in-scikit-learn
def mean_absolute_percentage_error(y_true, y_pred):
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

working_df = pd.read_csv('Data\\$working_data_5c.csv')

# eliminate some outliers, homes above an estimated value of $2 million are especially difficult to model
# with the available data
working_df = working_df[working_df['Age_Yrs'] > 0]
working_df = working_df[working_df['totalActualVal'] <= 2000000]

y = working_df['price']
columns = working_df.columns[2:]
X = working_df.drop(columns, axis=1)  # , 'totalActualVal'
X = X.drop(labels=['price'], axis=1)
# 70/30 split of data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=245)

# determine metrics
gradient, intercept, r_value, p_value, std_err = stats.linregress(X_test['totalActualVal'], y_test)

print 'Gradient: %.4f' % gradient
print 'R Value: %.4f' % r_value
print 'R-Squared: %.4f' % r_value ** 2

# adjusted R-squared - https://www.easycalculation.com/statistics/learn-adjustedr2.php
r_sq_adj = 1 - ((1 - r_value ** 2) * (len(y_test) - 1) / (len(y_test) - X_train.shape[1] - 1))
print 'R-Squared Adjusted: %.4f' % r_sq_adj

mape = mean_absolute_percentage_error(y_test, X_test['totalActualVal'])
print 'MAPE: %.4f' % mape

# plot with regression lines, one for actual data, one to represent ideal answer
z = np.polyfit(X_test['totalActualVal'], y_test, 1)
print 'z'
print z
y_poly = [z[0] * x + z[1] for x in range(int(intercept), 3100000 + int(intercept), 100000)]
x_poly = [x for x in range(0, 3100000, 100000)]
y_perfect = [x for x in range(0, 3100000, 100000)]

plt.figure(0)
plt.plot(X_test, y_test, ".")
plt.plot(x_poly, y_poly, "-")
plt.plot(x_poly, y_perfect, "-")
plt.xlim(0, 4000000)
plt.ylim(0, 4000000)
plt.xlabel("Est Price")
plt.ylabel("Actual Price")
plt.title("Estimated vs. Actual Sales Price")
plt.show()
plt.close()

# delta_price = pd.Series((X_test['totalActualVal'] / y_test * 100.0) - 100.0)
# delta_price.to_csv('Data\\delta_price_basic.csv', index=False)

print 'min price, actual: %.2f' % np.min(y_test)
print 'min price, assessor estimate: %.2f' % np.min(X_test['totalActualVal'])


"""
Title: Semantic Similarity with BERT
Author: [Mohamad Merchant](https://twitter.com/mohmadmerchant1)
Date created: 2020/08/15
Last modified: 2020/08/29
Description: Natural Language Inference by fine-tuning BERT model on SNLI Corpus.
"""
"""
## Introduction

Semantic Similarity is the task of determining how similar
two sentences are, in terms of what they mean.
This example demonstrates the use of SNLI (Stanford Natural Language Inference) Corpus
to predict sentence semantic similarity with Transformers.
We will fine-tune a BERT model that takes two sentences as inputs
and that outputs a similarity score for these two sentences.

### References

* [BERT](https://arxiv.org/pdf/1810.04805.pdf)
* [SNLI](https://nlp.stanford.edu/projects/snli/)
"""

"""
## Setup

Note: install HuggingFace `transformers` via `pip install transformers` (version >= 2.11.0).
"""
import numpy as np
import pandas as pd
import tensorflow as tf
import transformers

"""
## Configuration
"""

max_length = 128  # Maximum length of input sentence to the model.
batch_size = 32
epochs = 2

# Labels in our dataset.
labels = ["contradiction", "entailment", "neutral"]

"""
## Load the Data
"""

"""shell
curl -LO https://raw.githubusercontent.com/MohamadMerchant/SNLI/master/data.tar.gz
tar -xvzf data.tar.gz
"""
# There are more than 550k samples in total; we will use 100k for this example.
train_df = pd.read_csv("SNLI_Corpus/snli_1.0_train.csv", nrows=100000)
valid_df = pd.read_csv("SNLI_Corpus/snli_1.0_dev.csv")
test_df = pd.read_csv("SNLI_Corpus/snli_1.0_test.csv")

# Shape of the data
print(f"Total train samples : {train_df.shape[0]}")
print(f"Total validation samples: {valid_df.shape[0]}")
print(f"Total test samples: {valid_df.shape[0]}")

"""
Dataset Overview:

- sentence1: The premise caption that was supplied to the author of the pair.
- sentence2: The hypothesis caption that was written by the author of the pair.
- similarity: This is the label chosen by the majority of annotators.
Where no majority exists, the label "-" is used (we will skip such samples here).

Here are the "similarity" label values in our dataset:

- Contradiction: The sentences share no similarity.
- Entailment: The sentences have similar meaning.
- Neutral: The sentences are neutral.
"""

"""
Let's look at one sample from the dataset:
"""
print(f"Sentence1: {train_df.loc[1, 'sentence1']}")
print(f"Sentence2: {train_df.loc[1, 'sentence2']}")
print(f"Similarity: {train_df.loc[1, 'similarity']}")

"""
## Preprocessing
"""

# We have some NaN entries in our train data, we will simply drop them.
print("Number of missing values")
print(train_df.isnull().sum())
train_df.dropna(axis=0, inplace=True)

"""
Distribution of our training targets.
"""
print("Train Target Distribution")
print(train_df.similarity.value_counts())

"""
Distribution of our validation targets.
"""
print("Validation Target Distribution")
print(valid_df.similarity.value_counts())

"""
The value "-" appears as part of our training and validation targets.
We will skip these samples.
"""
train_df = (
    train_df[train_df.similarity != "-"]
    .sample(frac=1.0, random_state=42)
    .reset_index(drop=True)
)
valid_df = (
    valid_df[valid_df.similarity != "-"]
    .sample(frac=1.0, random_state=42)
    .reset_index(drop=True)
)

"""
One-hot encode training, validation, and test labels.
"""
train_df["label"] = train_df["similarity"].apply(
    lambda x: 0 if x == "contradiction" else 1 if x == "entailment" else 2
)
y_train = tf.keras.utils.to_categorical(train_df.label, num_classes=3)

valid_df["label"] = valid_df["similarity"].apply(
    lambda x: 0 if x == "contradiction" else 1 if x == "entailment" else 2
)
y_val = tf.keras.utils.to_categorical(valid_df.label, num_classes=3)

test_df["label"] = test_df["similarity"].apply(
    lambda x: 0 if x == "contradiction" else 1 if x == "entailment" else 2
)
y_test = tf.keras.utils.to_categorical(test_df.label, num_classes=3)

"""
## Create a custom data generator
"""

class BertSemanticDataGenerator(tf.keras.utils.Sequence):
    """Generates batches of data.

    Args:
        sentence_pairs: Array of premise and hypothesis input sentences.
        labels: Array of labels.
        batch_size: Integer batch size.
        shuffle: boolean, whether to shuffle the data.
        include_targets: boolean, whether to incude the labels.

    Returns:
        Tuples `([input_ids, attention_mask, `token_type_ids], labels)`
        (or just `[input_ids, attention_mask, `token_type_ids]`
         if `include_targets=False`)
    """

    def __init__(
        self,
        sentence_pairs,
        labels,
        batch_size=batch_size,
        shuffle=True,
        include_targets=True,
    ):
        self.sentence_pairs = sentence_pairs
        self.labels = labels
        self.shuffle = shuffle
        self.batch_size = batch_size
        self.include_targets = include_targets
        # Load our BERT Tokenizer to encode the text.
        # We will use base-base-uncased pretrained model.
        self.tokenizer = transformers.BertTokenizer.from_pretrained(
            "bert-base-uncased", do_lower_case=True
        )
        self.indexes = np.arange(len(self.sentence_pairs))
        self.on_epoch_end()

    def __len__(self):
        # Denotes the number of batches per epoch.
        return len(self.sentence_pairs) // self.batch_size

    def __getitem__(self, idx):
        # Retrieves the batch of index.
        indexes = self.indexes[idx * self.batch_size : (idx + 1) * self.batch_size]
        sentence_pairs = self.sentence_pairs[indexes]

        # With BERT tokenizer's batch_encode_plus batch of both the sentences are
        # encoded together and separated by [SEP] token.
        encoded = self.tokenizer.batch_encode_plus(
            sentence_pairs.tolist(),
            add_special_tokens=True,
            max_length=max_length,
            return_attention_mask=True,
            return_token_type_ids=True,
            pad_to_max_length=True,
            return_tensors="tf",
        )

        # Convert batch of encoded features to numpy array.
        input_ids = np.array(encoded["input_ids"], dtype="int32")
        attention_masks = np.array(encoded["attention_mask"], dtype="int32")
        token_type_ids = np.array(encoded["token_type_ids"], dtype="int32")

        # Set to true if data generator is used for training/validation.
        if self.include_targets:
            labels = np.array(self.labels[indexes], dtype="int32")
            return [input_ids, attention_masks, token_type_ids], labels
        else:
            return [input_ids, attention_masks, token_type_ids]

    def on_epoch_end(self):
        # Shuffle indexes after each epoch if shuffle is set to True.
        if self.shuffle:
            np.random.RandomState(42).shuffle(self.indexes)

"""
## Build the model
"""
# Create the model under a distribution strategy scope.
strategy = tf.distribute.MirroredStrategy()

with strategy.scope():
    # Encoded token ids from BERT tokenizer.
    input_ids = tf.keras.layers.Input(
        shape=(max_length,), dtype=tf.int32, name="input_ids"
    )
    # Attention masks indicates to the model which tokens should be attended to.
    attention_masks = tf.keras.layers.Input(
        shape=(max_length,), dtype=tf.int32, name="attention_masks"
    )
    # Token type ids are binary masks identifying different sequences in the model.
    token_type_ids = tf.keras.layers.Input(
        shape=(max_length,), dtype=tf.int32, name="token_type_ids"
    )
    # Loading pretrained BERT model.
    bert_model = transformers.TFBertModel.from_pretrained("bert-base-uncased")
    # Freeze the BERT model to reuse the pretrained features without modifying them.
    bert_model.trainable = False

    sequence_output, pooled_output = bert_model(
        input_ids, attention_mask=attention_masks, token_type_ids=token_type_ids
    )
    # Add trainable layers on top of frozen layers to adapt the pretrained features on the new data.
    bi_lstm = tf.keras.layers.Bidirectional(
        tf.keras.layers.LSTM(64, return_sequences=True)
    )(sequence_output)
    # Applying hybrid pooling approach to bi_lstm sequence output.
    avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)
    max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)
    concat = tf.keras.layers.concatenate([avg_pool, max_pool])
    dropout = tf.keras.layers.Dropout(0.3)(concat)
    output = tf.keras.layers.Dense(3, activation="softmax")(dropout)
    model = tf.keras.models.Model(
        inputs=[input_ids, attention_masks, token_type_ids], outputs=output
    )

    model.compile(
        optimizer=tf.keras.optimizers.Adam(),
        loss="categorical_crossentropy",
        metrics=["acc"],
    )

print(f"Strategy: {strategy}")
model.summary()

"""
Create train and validation data generators
"""
train_data = BertSemanticDataGenerator(
    train_df[["sentence1", "sentence2"]].values.astype("str"),
    y_train,
    batch_size=batch_size,
    shuffle=True,
)
valid_data = BertSemanticDataGenerator(
    valid_df[["sentence1", "sentence2"]].values.astype("str"),
    y_val,
    batch_size=batch_size,
    shuffle=False,
)

"""
## Train the Model

Training is done only for the top layers to perform "feature extraction",
which will allow the model to use the representations of the pretrained model.
"""
history = model.fit(
    train_data,
    validation_data=valid_data,
    epochs=epochs,
    use_multiprocessing=True,
    workers=-1,
)

"""
## Fine-tuning

This step must only be performed after the feature extraction model has
been trained to convergence on the new data.

This is an optional last step where `bert_model` is unfreezed and retrained
with a very low learning rate. This can deliver meaningful improvement by
incrementally adapting the pretrained features to the new data.
"""

# Unfreeze the bert_model.
bert_model.trainable = True
# Recompile the model to make the change effective.
model.compile(
    optimizer=tf.keras.optimizers.Adam(1e-5),
    loss="categorical_crossentropy",
    metrics=["accuracy"],
)
model.summary()

"""
## Train the entire model end-to-end
"""
history = model.fit(
    train_data,
    validation_data=valid_data,
    epochs=epochs,
    use_multiprocessing=True,
    workers=-1,
)

"""
## Evaluate model on the test set
"""
test_data = BertSemanticDataGenerator(
    test_df[["sentence1", "sentence2"]].values.astype("str"),
    y_test,
    batch_size=batch_size,
    shuffle=False,
)
model.evaluate(test_data, verbose=1)

"""
## Inference on custom sentences
"""

def check_similarity(sentence1, sentence2):
    sentence_pairs = np.array([[str(sentence1), str(sentence2)]])
    test_data = BertSemanticDataGenerator(
        sentence_pairs, labels=None, batch_size=1, shuffle=False, include_targets=False,
    )

    proba = model.predict(test_data)[0]
    idx = np.argmax(proba)
    proba = f"{proba[idx]: .2f}%"
    pred = labels[idx]
    return pred, proba

"""
Check results on some example sentence pairs.
"""
sentence1 = "Two women are observing something together."
sentence2 = "Two women are standing with their eyes closed."
check_similarity(sentence1, sentence2)
"""
Check results on some example sentence pairs.
"""
sentence1 = "A smiling costumed woman is holding an umbrella"
sentence2 = "A happy woman in a fairy costume holds an umbrella"
check_similarity(sentence1, sentence2)

"""
Check results on some example sentence pairs
"""
sentence1 = "A soccer game with multiple males playing"
sentence2 = "Some men are playing a sport"
check_similarity(sentence1, sentence2)


# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.

from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import range
import datetime as dt
import collections
import mock
from oauth2client.client import AccessTokenCredentials
import pandas
import unittest

# import Python so we can mock the parts we need to here.
import IPython
import IPython.core.magic

IPython.core.magic.register_line_cell_magic = mock.Mock()
IPython.core.magic.register_line_magic = mock.Mock()
IPython.core.magic.register_cell_magic = mock.Mock()
IPython.get_ipython = mock.Mock()

import datalab.bigquery  # noqa: E402
import datalab.context  # noqa: E402
import datalab.utils.commands  # noqa: E402

class TestCases(unittest.TestCase):

  @staticmethod
  def _get_expected_cols():
    cols = [
      {'type': 'number', 'id': 'Column1', 'label': 'Column1'},
      {'type': 'number', 'id': 'Column2', 'label': 'Column2'},
      {'type': 'string', 'id': 'Column3', 'label': 'Column3'},
      {'type': 'boolean', 'id': 'Column4', 'label': 'Column4'},
      {'type': 'number', 'id': 'Column5', 'label': 'Column5'},
      {'type': 'datetime', 'id': 'Column6', 'label': 'Column6'}
    ]
    return cols

  @staticmethod
  def _timestamp(d):
    return (d - dt.datetime(1970, 1, 1)).total_seconds()

  @staticmethod
  def _get_raw_rows():
    rows = [
      {'f': [
        {'v': 1}, {'v': 2}, {'v': '3'}, {'v': 'true'}, {'v': 0.0},
        {'v': TestCases._timestamp(dt.datetime(2000, 1, 1))}
      ]},
      {'f': [
        {'v': 11}, {'v': 12}, {'v': '13'}, {'v': 'false'}, {'v': 0.2},
        {'v': TestCases._timestamp(dt.datetime(2000, 1, 2))}
      ]},
      {'f': [
        {'v': 21}, {'v': 22}, {'v': '23'}, {'v': 'true'}, {'v': 0.3},
        {'v': TestCases._timestamp(dt.datetime(2000, 1, 3))}
      ]},
      {'f': [
        {'v': 31}, {'v': 32}, {'v': '33'}, {'v': 'false'}, {'v': 0.4},
        {'v': TestCases._timestamp(dt.datetime(2000, 1, 4))}
      ]},
      {'f': [
        {'v': 41}, {'v': 42}, {'v': '43'}, {'v': 'true'}, {'v': 0.5},
        {'v': TestCases._timestamp(dt.datetime(2000, 1, 5))}
      ]},
      {'f': [
        {'v': 51}, {'v': 52}, {'v': '53'}, {'v': 'true'}, {'v': 0.6},
        {'v': TestCases._timestamp(dt.datetime(2000, 1, 6))}
      ]}
    ]
    return rows

  @staticmethod
  def _get_expected_rows():
    rows = [
      {'c': [
        {'v': 1}, {'v': 2}, {'v': '3'}, {'v': True}, {'v': 0.0}, {'v': dt.datetime(2000, 1, 1)}
      ]},
      {'c': [
        {'v': 11}, {'v': 12}, {'v': '13'}, {'v': False}, {'v': 0.2}, {'v': dt.datetime(2000, 1, 2)}
      ]},
      {'c': [
        {'v': 21}, {'v': 22}, {'v': '23'}, {'v': True}, {'v': 0.3}, {'v': dt.datetime(2000, 1, 3)}
      ]},
      {'c': [
        {'v': 31}, {'v': 32}, {'v': '33'}, {'v': False}, {'v': 0.4}, {'v': dt.datetime(2000, 1, 4)}
      ]},
      {'c': [
        {'v': 41}, {'v': 42}, {'v': '43'}, {'v': True}, {'v': 0.5}, {'v': dt.datetime(2000, 1, 5)}
      ]},
      {'c': [
        {'v': 51}, {'v': 52}, {'v': '53'}, {'v': True}, {'v': 0.6}, {'v': dt.datetime(2000, 1, 6)}
      ]}
    ]
    return rows

  @staticmethod
  def _get_test_data_as_list_of_dicts():
    test_data = [
      {'Column1': 1, 'Column2': 2, 'Column3': '3',
       'Column4': True, 'Column5': 0.0, 'Column6': dt.datetime(2000, 1, 1)},
      {'Column1': 11, 'Column2': 12, 'Column3': '13',
       'Column4': False, 'Column5': 0.2, 'Column6': dt.datetime(2000, 1, 2)},
      {'Column1': 21, 'Column2': 22, 'Column3': '23',
       'Column4': True, 'Column5': 0.3, 'Column6': dt.datetime(2000, 1, 3)},
      {'Column1': 31, 'Column2': 32, 'Column3': '33',
       'Column4': False, 'Column5': 0.4, 'Column6': dt.datetime(2000, 1, 4)},
      {'Column1': 41, 'Column2': 42, 'Column3': '43',
       'Column4': True, 'Column5': 0.5, 'Column6': dt.datetime(2000, 1, 5)},
      {'Column1': 51, 'Column2': 52, 'Column3': '53',
       'Column4': True, 'Column5': 0.6, 'Column6': dt.datetime(2000, 1, 6)}
    ]
    # Use OrderedDicts to make testing the result easier.
    for i in range(0, len(test_data)):
      test_data[i] = collections.OrderedDict(sorted(list(test_data[i].items()), key=lambda t: t[0]))

    return test_data

  def test_get_data_from_list_of_dicts(self):
    self._test_get_data(TestCases._get_test_data_as_list_of_dicts(), TestCases._get_expected_cols(),
                        TestCases._get_expected_rows(), 6,
                        datalab.utils.commands._utils._get_data_from_list_of_dicts)
    self._test_get_data(TestCases._get_test_data_as_list_of_dicts(), TestCases._get_expected_cols(),
                        TestCases._get_expected_rows(), 6, datalab.utils.commands._utils.get_data)

  def test_get_data_from_list_of_lists(self):
    test_data = [
      [1, 2, '3', True, 0.0, dt.datetime(2000, 1, 1)],
      [11, 12, '13', False, 0.2, dt.datetime(2000, 1, 2)],
      [21, 22, '23', True, 0.3, dt.datetime(2000, 1, 3)],
      [31, 32, '33', False, 0.4, dt.datetime(2000, 1, 4)],
      [41, 42, '43', True, 0.5, dt.datetime(2000, 1, 5)],
      [51, 52, '53', True, 0.6, dt.datetime(2000, 1, 6)],
    ]

    self._test_get_data(test_data, TestCases._get_expected_cols(), TestCases._get_expected_rows(),
                        6, datalab.utils.commands._utils._get_data_from_list_of_lists)
    self._test_get_data(test_data, TestCases._get_expected_cols(), TestCases._get_expected_rows(),
                        6, datalab.utils.commands._utils.get_data)

  def test_get_data_from_dataframe(self):
    df = pandas.DataFrame(self._get_test_data_as_list_of_dicts())
    self._test_get_data(df, TestCases._get_expected_cols(), TestCases._get_expected_rows(), 6,
                        datalab.utils.commands._utils._get_data_from_dataframe)
    self._test_get_data(df, TestCases._get_expected_cols(), TestCases._get_expected_rows(), 6,
                        datalab.utils.commands._utils.get_data)

  @mock.patch('datalab.bigquery._api.Api.tabledata_list')
  @mock.patch('datalab.bigquery._table.Table.exists')
  @mock.patch('datalab.bigquery._api.Api.tables_get')
  @mock.patch('datalab.context._context.Context.default')
  def test_get_data_from_table(self, mock_context_default, mock_api_tables_get,
                               mock_table_exists, mock_api_tabledata_list):
    data = TestCases._get_expected_rows()
    mock_context_default.return_value = TestCases._create_context()
    mock_api_tables_get.return_value = {
      'numRows': len(data),
      'schema': {
        'fields': [
          {'name': 'Column1', 'type': 'INTEGER'},
          {'name': 'Column2', 'type': 'INTEGER'},
          {'name': 'Column3', 'type': 'STRING'},
          {'name': 'Column4', 'type': 'BOOLEAN'},
          {'name': 'Column5', 'type': 'FLOAT'},
          {'name': 'Column6', 'type': 'TIMESTAMP'}
        ]
      }
    }
    mock_table_exists.return_value = True
    raw_data = self._get_raw_rows()

    def tabledata_list(*args, **kwargs):
      start_index = kwargs['start_index']
      max_results = kwargs['max_results']
      if max_results < 0:
        max_results = len(data)
      return {'rows': raw_data[start_index:start_index + max_results]}

    mock_api_tabledata_list.side_effect = tabledata_list
    t = datalab.bigquery.Table('foo.bar')
    self._test_get_data(t, TestCases._get_expected_cols(), TestCases._get_expected_rows(), 6,
                        datalab.utils.commands._utils._get_data_from_table)
    self._test_get_data(t, TestCases._get_expected_cols(), TestCases._get_expected_rows(), 6,
                        datalab.utils.commands._utils.get_data)

  def test_get_data_from_empty_list(self):
    self._test_get_data([], [], [], 0, datalab.utils.commands._utils.get_data)

  def test_get_data_from_malformed_list(self):
    with self.assertRaises(Exception) as error:
      self._test_get_data(['foo', 'bar'], [], [], 0, datalab.utils.commands._utils.get_data)
    self.assertEquals('To get tabular data from a list it must contain dictionaries or lists.',
                      str(error.exception))

  def _test_get_data(self, test_data, cols, rows, expected_count, fn):
    self.maxDiff = None
    data, count = fn(test_data)
    self.assertEquals(expected_count, count)
    self.assertEquals({'cols': cols, 'rows': rows}, data)

    # Test first_row. Note that count must be set in this case so we use a value greater than the
    # data set size.
    for first in range(0, 6):
      data, count = fn(test_data, first_row=first, count=10)
      self.assertEquals(expected_count, count)
      self.assertEquals({'cols': cols, 'rows': rows[first:]}, data)

    # Test first_row + count
    for first in range(0, 6):
      data, count = fn(test_data, first_row=first, count=2)
      self.assertEquals(expected_count, count)
      self.assertEquals({'cols': cols, 'rows': rows[first:first + 2]}, data)

    # Test subsets of columns

    # No columns
    data, count = fn(test_data, fields=[])
    self.assertEquals({'cols': [], 'rows': [{'c': []}] * expected_count}, data)

    # Single column
    data, count = fn(test_data, fields=['Column3'])

    if expected_count == 0:
      return

    self.assertEquals({'cols': [cols[2]],
                       'rows': [{'c': [row['c'][2]]} for row in rows]}, data)

    # Multi-columns
    data, count = fn(test_data, fields=['Column1', 'Column3', 'Column6'])
    self.assertEquals({'cols': [cols[0], cols[2], cols[5]],
                       'rows': [{'c': [row['c'][0], row['c'][2], row['c'][5]]} for row in rows]},
                      data)

    # Switch order
    data, count = fn(test_data, fields=['Column3', 'Column1'])
    self.assertEquals({'cols': [cols[2], cols[0]],
                       'rows': [{'c': [row['c'][2], row['c'][0]]} for row in rows]}, data)

    # Select all
    data, count = fn(test_data,
                     fields=['Column1', 'Column2', 'Column3', 'Column4', 'Column5', 'Column6'])
    self.assertEquals({'cols': cols, 'rows': rows}, data)

  @staticmethod
  def _create_api():
    context = TestCases._create_context()
    return datalab.bigquery._api.Api(context.credentials, context.project_id)

  @staticmethod
  def _create_context():
    project_id = 'test'
    creds = AccessTokenCredentials('test_token', 'test_ua')
    return datalab.context.Context(project_id, creds)