"""
=========================================================
Comparing different clustering algorithms on toy datasets
=========================================================

This example aims at showing characteristics of different
clustering algorithms on datasets that are "interesting"
but still in 2D. The last dataset is an example of a 'null'
situation for clustering: the data is homogeneous, and
there is no good clustering.

While these examples give some intuition about the algorithms,
this intuition might not apply to very high dimensional data.

The results could be improved by tweaking the parameters for
each clustering strategy, for instance setting the number of
clusters for the methods that needs this parameter
specified. Note that affinity propagation has a tendency to
create many clusters. Thus in this example its two parameters
(damping and per-point preference) were set to mitigate this
behavior.
"""
print(__doc__)

import time

import numpy as np
import matplotlib.pyplot as plt

from sklearn import cluster, datasets
from sklearn.neighbors import kneighbors_graph
from sklearn.preprocessing import StandardScaler

np.random.seed(0)

# Generate datasets. We choose the size big enough to see the scalability
# of the algorithms, but not too big to avoid too long running times
n_samples = 1500
noisy_circles = datasets.make_circles(n_samples=n_samples, factor=.5,
                                      noise=.05)
noisy_moons = datasets.make_moons(n_samples=n_samples, noise=.05)
blobs = datasets.make_blobs(n_samples=n_samples, random_state=8)
no_structure = np.random.rand(n_samples, 2), None

colors = np.array([x for x in 'bgrcmykbgrcmykbgrcmykbgrcmyk'])
colors = np.hstack([colors] * 20)

clustering_names = [
    'MiniBatchKMeans', 'AffinityPropagation', 'MeanShift',
    'SpectralClustering', 'Ward', 'AgglomerativeClustering',
    'DBSCAN', 'Birch']

plt.figure(figsize=(len(clustering_names) * 2 + 3, 9.5))
plt.subplots_adjust(left=.02, right=.98, bottom=.001, top=.96, wspace=.05,
                    hspace=.01)

plot_num = 1

datasets = [noisy_circles, noisy_moons, blobs, no_structure]
for i_dataset, dataset in enumerate(datasets):
    X, y = dataset
    # normalize dataset for easier parameter selection
    X = StandardScaler().fit_transform(X)

    # estimate bandwidth for mean shift
    bandwidth = cluster.estimate_bandwidth(X, quantile=0.3)

    # connectivity matrix for structured Ward
    connectivity = kneighbors_graph(X, n_neighbors=10, include_self=False)
    # make connectivity symmetric
    connectivity = 0.5 * (connectivity + connectivity.T)

    # create clustering estimators
    ms = cluster.MeanShift(bandwidth=bandwidth, bin_seeding=True)
    two_means = cluster.MiniBatchKMeans(n_clusters=2)
    ward = cluster.AgglomerativeClustering(n_clusters=2, linkage='ward',
                                           connectivity=connectivity)
    spectral = cluster.SpectralClustering(n_clusters=2,
                                          eigen_solver='arpack',
                                          affinity="nearest_neighbors")
    dbscan = cluster.DBSCAN(eps=.2)
    affinity_propagation = cluster.AffinityPropagation(damping=.9,
                                                       preference=-200)

    average_linkage = cluster.AgglomerativeClustering(
        linkage="average", affinity="cityblock", n_clusters=2,
        connectivity=connectivity)

    birch = cluster.Birch(n_clusters=2)
    clustering_algorithms = [
        two_means, affinity_propagation, ms, spectral, ward, average_linkage,
        dbscan, birch]

    for name, algorithm in zip(clustering_names, clustering_algorithms):
        # predict cluster memberships
        t0 = time.time()
        algorithm.fit(X)
        t1 = time.time()
        if hasattr(algorithm, 'labels_'):
            y_pred = algorithm.labels_.astype(np.int)
        else:
            y_pred = algorithm.predict(X)

        # plot
        plt.subplot(4, len(clustering_algorithms), plot_num)
        if i_dataset == 0:
            plt.title(name, size=18)
        plt.scatter(X[:, 0], X[:, 1], color=colors[y_pred].tolist(), s=10)

        if hasattr(algorithm, 'cluster_centers_'):
            centers = algorithm.cluster_centers_
            center_colors = colors[:len(centers)]
            plt.scatter(centers[:, 0], centers[:, 1], s=100, c=center_colors)
        plt.xlim(-2, 2)
        plt.ylim(-2, 2)
        plt.xticks(())
        plt.yticks(())
        plt.text(.99, .01, ('%.2fs' % (t1 - t0)).lstrip('0'),
                 transform=plt.gca().transAxes, size=15,
                 horizontalalignment='right')
        plot_num += 1

plt.show()


"""
Benchmarks of Lasso vs LassoLars

First, we fix a training set and increase the number of
samples. Then we plot the computation time as function of
the number of samples.

In the second benchmark, we increase the number of dimensions of the
training set. Then we plot the computation time as function of
the number of dimensions.

In both cases, only 10% of the features are informative.
"""
import gc
from time import time
import numpy as np

from sklearn.datasets.samples_generator import make_regression

def compute_bench(alpha, n_samples, n_features, precompute):
    lasso_results = []
    lars_lasso_results = []

    it = 0

    for ns in n_samples:
        for nf in n_features:
            it += 1
            print('==================')
            print('Iteration %s of %s' % (it, max(len(n_samples),
                                          len(n_features))))
            print('==================')
            n_informative = nf // 10
            X, Y, coef_ = make_regression(n_samples=ns, n_features=nf,
                                          n_informative=n_informative,
                                          noise=0.1, coef=True)

            X /= np.sqrt(np.sum(X ** 2, axis=0))  # Normalize data

            gc.collect()
            print("- benchmarking Lasso")
            clf = Lasso(alpha=alpha, fit_intercept=False,
                        precompute=precompute)
            tstart = time()
            clf.fit(X, Y)
            lasso_results.append(time() - tstart)

            gc.collect()
            print("- benchmarking LassoLars")
            clf = LassoLars(alpha=alpha, fit_intercept=False,
                            normalize=False, precompute=precompute)
            tstart = time()
            clf.fit(X, Y)
            lars_lasso_results.append(time() - tstart)

    return lasso_results, lars_lasso_results

if __name__ == '__main__':
    from sklearn.linear_model import Lasso, LassoLars
    import pylab as pl

    alpha = 0.01  # regularization parameter

    n_features = 10
    list_n_samples = np.linspace(100, 1000000, 5).astype(np.int)
    lasso_results, lars_lasso_results = compute_bench(alpha, list_n_samples,
                                            [n_features], precompute=True)

    pl.figure('scikit-learn LASSO benchmark results')
    pl.subplot(211)
    pl.plot(list_n_samples, lasso_results, 'b-',
                            label='Lasso')
    pl.plot(list_n_samples, lars_lasso_results, 'r-',
                            label='LassoLars')
    pl.title('precomputed Gram matrix, %d features, alpha=%s' % (n_features, alpha))
    pl.legend(loc='upper left')
    pl.xlabel('number of samples')
    pl.ylabel('Time (s)')
    pl.axis('tight')

    n_samples = 2000
    list_n_features = np.linspace(500, 3000, 5).astype(np.int)
    lasso_results, lars_lasso_results = compute_bench(alpha, [n_samples],
                                           list_n_features, precompute=False)
    pl.subplot(212)
    pl.plot(list_n_features, lasso_results, 'b-', label='Lasso')
    pl.plot(list_n_features, lars_lasso_results, 'r-', label='LassoLars')
    pl.title('%d samples, alpha=%s' % (n_samples, alpha))
    pl.legend(loc='upper left')
    pl.xlabel('number of features')
    pl.ylabel('Time (s)')
    pl.axis('tight')
    pl.show()


"""
GUI progressbar decorator for iterators.
Includes a default (x)range iterator printing to stderr.

Usage:
  >>> from tqdm_gui import tgrange[, tqdm_gui]
  >>> for i in tgrange(10): #same as: for i in tqdm_gui(xrange(10))
  ...     ...
"""
# future division is important to divide integers and get as
# a result precise floating numbers (instead of truncated int)
from __future__ import division, absolute_import
# import compatibility functions and utilities
# import sys
from time import time
from ._utils import _range
# to inherit from the tqdm class
from ._tqdm import tqdm, TqdmExperimentalWarning
from warnings import warn

__author__ = {"github.com/": ["casperdcl", "lrq3000"]}
__all__ = ['tqdm_gui', 'tgrange']

class tqdm_gui(tqdm):  # pragma: no cover
    """
    Experimental GUI version of tqdm!
    """

    # TODO: @classmethod: write() on GUI?

    def __init__(self, *args, **kwargs):
        import matplotlib as mpl
        import matplotlib.pyplot as plt
        from collections import deque
        kwargs['gui'] = True

        super(tqdm_gui, self).__init__(*args, **kwargs)

        # Initialize the GUI display
        if self.disable or not kwargs['gui']:
            return

        warn('GUI is experimental/alpha', TqdmExperimentalWarning)
        self.mpl = mpl
        self.plt = plt
        self.sp = None

        # Remember if external environment uses toolbars
        self.toolbar = self.mpl.rcParams['toolbar']
        self.mpl.rcParams['toolbar'] = 'None'

        self.mininterval = max(self.mininterval, 0.5)
        self.fig, ax = plt.subplots(figsize=(9, 2.2))
        # self.fig.subplots_adjust(bottom=0.2)
        if self.total:
            self.xdata = []
            self.ydata = []
            self.zdata = []
        else:
            self.xdata = deque([])
            self.ydata = deque([])
            self.zdata = deque([])
        self.line1, = ax.plot(self.xdata, self.ydata, color='b')
        self.line2, = ax.plot(self.xdata, self.zdata, color='k')
        ax.set_ylim(0, 0.001)
        if self.total:
            ax.set_xlim(0, 100)
            ax.set_xlabel('percent')
            self.fig.legend((self.line1, self.line2), ('cur', 'est'),
                            loc='center right')
            # progressbar
            self.hspan = plt.axhspan(0, 0.001,
                                     xmin=0, xmax=0, color='g')
        else:
            # ax.set_xlim(-60, 0)
            ax.set_xlim(0, 60)
            ax.invert_xaxis()
            ax.set_xlabel('seconds')
            ax.legend(('cur', 'est'), loc='lower left')
        ax.grid()
        # ax.set_xlabel('seconds')
        ax.set_ylabel((self.unit if self.unit else 'it') + '/s')
        if self.unit_scale:
            plt.ticklabel_format(style='sci', axis='y',
                                 scilimits=(0, 0))
            ax.yaxis.get_offset_text().set_x(-0.15)

        # Remember if external environment is interactive
        self.wasion = plt.isinteractive()
        plt.ion()
        self.ax = ax

    def __iter__(self):
        # TODO: somehow allow the following:
        # if not self.gui:
        #   return super(tqdm_gui, self).__iter__()
        iterable = self.iterable
        if self.disable:
            for obj in iterable:
                yield obj
            return

        # ncols = self.ncols
        mininterval = self.mininterval
        maxinterval = self.maxinterval
        miniters = self.miniters
        dynamic_miniters = self.dynamic_miniters
        unit = self.unit
        unit_scale = self.unit_scale
        ascii = self.ascii
        start_t = self.start_t
        last_print_t = self.last_print_t
        last_print_n = self.last_print_n
        n = self.n
        # dynamic_ncols = self.dynamic_ncols
        smoothing = self.smoothing
        avg_time = self.avg_time
        bar_format = self.bar_format

        plt = self.plt
        ax = self.ax
        xdata = self.xdata
        ydata = self.ydata
        zdata = self.zdata
        line1 = self.line1
        line2 = self.line2

        for obj in iterable:
            yield obj
            # Update and print the progressbar.
            # Note: does not call self.update(1) for speed optimisation.
            n += 1
            delta_it = n - last_print_n
            # check the counter first (avoid calls to time())
            if delta_it >= miniters:
                cur_t = time()
                delta_t = cur_t - last_print_t
                if delta_t >= mininterval:
                    elapsed = cur_t - start_t
                    # EMA (not just overall average)
                    if smoothing and delta_t:
                        avg_time = delta_t / delta_it \
                            if avg_time is None \
                            else smoothing * delta_t / delta_it + \
                            (1 - smoothing) * avg_time

                    # Inline due to multiple calls
                    total = self.total
                    # instantaneous rate
                    y = delta_it / delta_t
                    # overall rate
                    z = n / elapsed
                    # update line data
                    xdata.append(n * 100.0 / total if total else cur_t)
                    ydata.append(y)
                    zdata.append(z)

                    # Discard old values
                    # xmin, xmax = ax.get_xlim()
                    # if (not total) and elapsed > xmin * 1.1:
                    if (not total) and elapsed > 66:
                        xdata.popleft()
                        ydata.popleft()
                        zdata.popleft()

                    ymin, ymax = ax.get_ylim()
                    if y > ymax or z > ymax:
                        ymax = 1.1 * y
                        ax.set_ylim(ymin, ymax)
                        ax.figure.canvas.draw()

                    if total:
                        line1.set_data(xdata, ydata)
                        line2.set_data(xdata, zdata)
                        try:
                            poly_lims = self.hspan.get_xy()
                        except AttributeError:
                            self.hspan = plt.axhspan(0, 0.001, xmin=0,
                                                     xmax=0, color='g')
                            poly_lims = self.hspan.get_xy()
                        poly_lims[0, 1] = ymin
                        poly_lims[1, 1] = ymax
                        poly_lims[2] = [n / total, ymax]
                        poly_lims[3] = [poly_lims[2, 0], ymin]
                        if len(poly_lims) > 4:
                            poly_lims[4, 1] = ymin
                        self.hspan.set_xy(poly_lims)
                    else:
                        t_ago = [cur_t - i for i in xdata]
                        line1.set_data(t_ago, ydata)
                        line2.set_data(t_ago, zdata)

                    ax.set_title(self.format_meter(
                        n, total, elapsed, 0,
                        self.desc, ascii, unit, unit_scale,
                        1 / avg_time if avg_time else None, bar_format),
                        fontname="DejaVu Sans Mono", fontsize=11)
                    plt.pause(1e-9)

                    # If no `miniters` was specified, adjust automatically
                    # to the maximum iteration rate seen so far.
                    if dynamic_miniters:
                        if maxinterval and delta_t > maxinterval:
                            # Set miniters to correspond to maxinterval
                            miniters = delta_it * maxinterval / delta_t
                        elif mininterval and delta_t:
                            # EMA-weight miniters to converge
                            # towards the timeframe of mininterval
                            miniters = smoothing * delta_it * mininterval \
                                / delta_t + (1 - smoothing) * miniters
                        else:
                            miniters = smoothing * delta_it + \
                                (1 - smoothing) * miniters

                    # Store old values for next call
                    last_print_n = n
                    last_print_t = cur_t

        # Closing the progress bar.
        # Update some internal variables for close().
        self.last_print_n = last_print_n
        self.n = n
        self.close()

    def update(self, n=1):
        # if not self.gui:
        #   return super(tqdm_gui, self).close()
        if self.disable:
            return

        if n < 0:
            n = 1
        self.n += n

        delta_it = self.n - self.last_print_n  # should be n?
        if delta_it >= self.miniters:
            # We check the counter first, to reduce the overhead of time()
            cur_t = time()
            delta_t = cur_t - self.last_print_t
            if delta_t >= self.mininterval:
                elapsed = cur_t - self.start_t
                # EMA (not just overall average)
                if self.smoothing and delta_t:
                    self.avg_time = delta_t / delta_it \
                        if self.avg_time is None \
                        else self.smoothing * delta_t / delta_it + \
                        (1 - self.smoothing) * self.avg_time

                # Inline due to multiple calls
                total = self.total
                ax = self.ax

                # instantaneous rate
                y = delta_it / delta_t
                # smoothed rate
                z = self.n / elapsed
                # update line data
                self.xdata.append(self.n * 100.0 / total
                                  if total else cur_t)
                self.ydata.append(y)
                self.zdata.append(z)

                # Discard old values
                if (not total) and elapsed > 66:
                    self.xdata.popleft()
                    self.ydata.popleft()
                    self.zdata.popleft()

                ymin, ymax = ax.get_ylim()
                if y > ymax or z > ymax:
                    ymax = 1.1 * y
                    ax.set_ylim(ymin, ymax)
                    ax.figure.canvas.draw()

                if total:
                    self.line1.set_data(self.xdata, self.ydata)
                    self.line2.set_data(self.xdata, self.zdata)
                    try:
                        poly_lims = self.hspan.get_xy()
                    except AttributeError:
                        self.hspan = self.plt.axhspan(0, 0.001, xmin=0,
                                                      xmax=0, color='g')
                        poly_lims = self.hspan.get_xy()
                    poly_lims[0, 1] = ymin
                    poly_lims[1, 1] = ymax
                    poly_lims[2] = [self.n / total, ymax]
                    poly_lims[3] = [poly_lims[2, 0], ymin]
                    if len(poly_lims) > 4:
                        poly_lims[4, 1] = ymin
                    self.hspan.set_xy(poly_lims)
                else:
                    t_ago = [cur_t - i for i in self.xdata]
                    self.line1.set_data(t_ago, self.ydata)
                    self.line2.set_data(t_ago, self.zdata)

                ax.set_title(self.format_meter(
                    self.n, total, elapsed, 0,
                    self.desc, self.ascii, self.unit, self.unit_scale,
                    1 / self.avg_time if self.avg_time else None,
                    self.bar_format),
                    fontname="DejaVu Sans Mono", fontsize=11)
                self.plt.pause(1e-9)

                # If no `miniters` was specified, adjust automatically to the
                # maximum iteration rate seen so far.
                # e.g.: After running `tqdm.update(5)`, subsequent
                # calls to `tqdm.update()` will only cause an update after
                # at least 5 more iterations.
                if self.dynamic_miniters:
                    if self.maxinterval and delta_t > self.maxinterval:
                        self.miniters = self.miniters * self.maxinterval \
                            / delta_t
                    elif self.mininterval and delta_t:
                        self.miniters = self.smoothing * delta_it \
                            * self.mininterval / delta_t + \
                            (1 - self.smoothing) * self.miniters
                    else:
                        self.miniters = self.smoothing * delta_it + \
                            (1 - self.smoothing) * self.miniters

                # Store old values for next call
                self.last_print_n = self.n
                self.last_print_t = cur_t

    def close(self):
        # if not self.gui:
        #   return super(tqdm_gui, self).close()
        if self.disable:
            return

        self.disable = True

        self._instances.remove(self)

        # Restore toolbars
        self.mpl.rcParams['toolbar'] = self.toolbar
        # Return to non-interactive mode
        if not self.wasion:
            self.plt.ioff()
        if not self.leave:
            self.plt.close(self.fig)

def tgrange(*args, **kwargs):
    """
    A shortcut for tqdm_gui(xrange(*args), **kwargs).
    On Python3+ range is used instead of xrange.
    """
    return tqdm_gui(_range(*args), **kwargs)


"""Utilities to evaluate the clustering performance of models

Functions named as *_score return a scalar value to maximize: the higher the
better.
"""

# Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Wei LI <kuantkid@gmail.com>
#          Diego Molla <dmolla-aliod@gmail.com>
# License: BSD 3 clause

from math import log

from scipy.misc import comb
from scipy.sparse import coo_matrix
import numpy as np

from .expected_mutual_info_fast import expected_mutual_information
from ...utils.fixes import bincount

def comb2(n):
    # the exact version is faster for k == 2: use it by default globally in
    # this module instead of the float approximate variant
    return comb(n, 2, exact=1)

def check_clusterings(labels_true, labels_pred):
    """Check that the two clusterings matching 1D integer arrays"""
    labels_true = np.asarray(labels_true)
    labels_pred = np.asarray(labels_pred)

    # input checks
    if labels_true.ndim != 1:
        raise ValueError(
            "labels_true must be 1D: shape is %r" % (labels_true.shape,))
    if labels_pred.ndim != 1:
        raise ValueError(
            "labels_pred must be 1D: shape is %r" % (labels_pred.shape,))
    if labels_true.shape != labels_pred.shape:
        raise ValueError(
            "labels_true and labels_pred must have same size, got %d and %d"
            % (labels_true.shape[0], labels_pred.shape[0]))
    return labels_true, labels_pred

def contingency_matrix(labels_true, labels_pred, eps=None):
    """Build a contengency matrix describing the relationship between labels.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        Ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        Cluster labels to evaluate

    eps: None or float
        If a float, that value is added to all values in the contingency
        matrix. This helps to stop NaN propagation.
        If ``None``, nothing is adjusted.

    Returns
    -------
    contingency: array, shape=[n_classes_true, n_classes_pred]
        Matrix :math:`C` such that :math:`C_{i, j}` is the number of samples in
        true class :math:`i` and in predicted class :math:`j`. If
        ``eps is None``, the dtype of this array will be integer. If ``eps`` is
        given, the dtype will be float.
    """
    classes, class_idx = np.unique(labels_true, return_inverse=True)
    clusters, cluster_idx = np.unique(labels_pred, return_inverse=True)
    n_classes = classes.shape[0]
    n_clusters = clusters.shape[0]
    # Using coo_matrix to accelerate simple histogram calculation,
    # i.e. bins are consecutive integers
    # Currently, coo_matrix is faster than histogram2d for simple cases
    contingency = coo_matrix((np.ones(class_idx.shape[0]),
                              (class_idx, cluster_idx)),
                             shape=(n_classes, n_clusters),
                             dtype=np.int).toarray()
    if eps is not None:
        # don't use += as contingency is integer
        contingency = contingency + eps
    return contingency

# clustering measures

def adjusted_rand_score(labels_true, labels_pred):
    """Rand index adjusted for chance

    The Rand Index computes a similarity measure between two clusterings
    by considering all pairs of samples and counting pairs that are
    assigned in the same or different clusters in the predicted and
    true clusterings.

    The raw RI score is then "adjusted for chance" into the ARI score
    using the following scheme::

        ARI = (RI - Expected_RI) / (max(RI) - Expected_RI)

    The adjusted Rand index is thus ensured to have a value close to
    0.0 for random labeling independently of the number of clusters and
    samples and exactly 1.0 when the clusterings are identical (up to
    a permutation).

    ARI is a symmetric measure::

        adjusted_rand_score(a, b) == adjusted_rand_score(b, a)

    Read more in the :ref:`User Guide <adjusted_rand_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        Ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        Cluster labels to evaluate

    Returns
    -------
    ari : float
       Similarity score between -1.0 and 1.0. Random labelings have an ARI
       close to 0.0. 1.0 stands for perfect match.

    Examples
    --------

    Perfectly maching labelings have a score of 1 even

      >>> from sklearn.metrics.cluster import adjusted_rand_score
      >>> adjusted_rand_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> adjusted_rand_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Labelings that assign all classes members to the same clusters
    are complete be not always pure, hence penalized::

      >>> adjusted_rand_score([0, 0, 1, 2], [0, 0, 1, 1])  # doctest: +ELLIPSIS
      0.57...

    ARI is symmetric, so labelings that have pure clusters with members
    coming from the same classes but unnecessary splits are penalized::

      >>> adjusted_rand_score([0, 0, 1, 1], [0, 0, 1, 2])  # doctest: +ELLIPSIS
      0.57...

    If classes members are completely split across different clusters, the
    assignment is totally incomplete, hence the ARI is very low::

      >>> adjusted_rand_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    References
    ----------

    .. [Hubert1985] `L. Hubert and P. Arabie, Comparing Partitions,
      Journal of Classification 1985`
      http://www.springerlink.com/content/x64124718341j1j0/

    .. [wk] http://en.wikipedia.org/wiki/Rand_index#Adjusted_Rand_index

    See also
    --------
    adjusted_mutual_info_score: Adjusted Mutual Information

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    n_samples = labels_true.shape[0]
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split;
    # or trivial clustering where each document is assigned a unique cluster.
    # These are perfect matches hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1
            or classes.shape[0] == clusters.shape[0] == 0
            or classes.shape[0] == clusters.shape[0] == len(labels_true)):
        return 1.0

    contingency = contingency_matrix(labels_true, labels_pred)

    # Compute the ARI using the contingency data
    sum_comb_c = sum(comb2(n_c) for n_c in contingency.sum(axis=1))
    sum_comb_k = sum(comb2(n_k) for n_k in contingency.sum(axis=0))

    sum_comb = sum(comb2(n_ij) for n_ij in contingency.flatten())
    prod_comb = (sum_comb_c * sum_comb_k) / float(comb(n_samples, 2))
    mean_comb = (sum_comb_k + sum_comb_c) / 2.
    return ((sum_comb - prod_comb) / (mean_comb - prod_comb))

def homogeneity_completeness_v_measure(labels_true, labels_pred):
    """Compute the homogeneity and completeness and V-Measure scores at once

    Those metrics are based on normalized conditional entropy measures of
    the clustering labeling to evaluate given the knowledge of a Ground
    Truth class labels of the same samples.

    A clustering result satisfies homogeneity if all of its clusters
    contain only data points which are members of a single class.

    A clustering result satisfies completeness if all the data points
    that are members of a given class are elements of the same cluster.

    Both scores have positive values between 0.0 and 1.0, larger values
    being desirable.

    Those 3 metrics are independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score values in any way.

    V-Measure is furthermore symmetric: swapping ``labels_true`` and
    ``label_pred`` will give the same score. This does not hold for
    homogeneity and completeness.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    homogeneity: float
       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling

    completeness: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    v_measure: float
        harmonic mean of the first two

    See also
    --------
    homogeneity_score
    completeness_score
    v_measure_score
    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)

    if len(labels_true) == 0:
        return 1.0, 1.0, 1.0

    entropy_C = entropy(labels_true)
    entropy_K = entropy(labels_pred)

    MI = mutual_info_score(labels_true, labels_pred)

    homogeneity = MI / (entropy_C) if entropy_C else 1.0
    completeness = MI / (entropy_K) if entropy_K else 1.0

    if homogeneity + completeness == 0.0:
        v_measure_score = 0.0
    else:
        v_measure_score = (2.0 * homogeneity * completeness
                           / (homogeneity + completeness))

    return homogeneity, completeness, v_measure_score

def homogeneity_score(labels_true, labels_pred):
    """Homogeneity metric of a cluster labeling given a ground truth

    A clustering result satisfies homogeneity if all of its clusters
    contain only data points which are members of a single class.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is not symmetric: switching ``label_true`` with ``label_pred``
    will return the :func:`completeness_score` which will be different in
    general.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    homogeneity: float
       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    completeness_score
    v_measure_score

    Examples
    --------

    Perfect labelings are homogeneous::

      >>> from sklearn.metrics.cluster import homogeneity_score
      >>> homogeneity_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Non-perfect labelings that further split classes into more clusters can be
    perfectly homogeneous::

      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 0, 1, 2]))
      ...                                                  # doctest: +ELLIPSIS
      1.0...
      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      1.0...

    Clusters that include samples from different classes do not make for an
    homogeneous labeling::

      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 1, 0, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...
      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 0, 0, 0]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred)[0]

def completeness_score(labels_true, labels_pred):
    """Completeness metric of a cluster labeling given a ground truth

    A clustering result satisfies completeness if all the data points
    that are members of a given class are elements of the same cluster.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is not symmetric: switching ``label_true`` with ``label_pred``
    will return the :func:`homogeneity_score` which will be different in
    general.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    completeness: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    homogeneity_score
    v_measure_score

    Examples
    --------

    Perfect labelings are complete::

      >>> from sklearn.metrics.cluster import completeness_score
      >>> completeness_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Non-perfect labelings that assign all classes members to the same clusters
    are still complete::

      >>> print(completeness_score([0, 0, 1, 1], [0, 0, 0, 0]))
      1.0
      >>> print(completeness_score([0, 1, 2, 3], [0, 0, 1, 1]))
      1.0

    If classes members are split across different clusters, the
    assignment cannot be complete::

      >>> print(completeness_score([0, 0, 1, 1], [0, 1, 0, 1]))
      0.0
      >>> print(completeness_score([0, 0, 0, 0], [0, 1, 2, 3]))
      0.0

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred)[1]

def v_measure_score(labels_true, labels_pred):
    """V-measure cluster labeling given a ground truth.

    This score is identical to :func:`normalized_mutual_info_score`.

    The V-measure is the harmonic mean between homogeneity and completeness::

        v = 2 * (homogeneity * completeness) / (homogeneity + completeness)

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    v_measure: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    homogeneity_score
    completeness_score

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have score 1.0::

      >>> from sklearn.metrics.cluster import v_measure_score
      >>> v_measure_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> v_measure_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Labelings that assign all classes members to the same clusters
    are complete be not homogeneous, hence penalized::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 2], [0, 0, 1, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.8...
      >>> print("%.6f" % v_measure_score([0, 1, 2, 3], [0, 0, 1, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.66...

    Labelings that have pure clusters with members coming from the same
    classes are homogeneous but un-necessary splits harms completeness
    and thus penalize V-measure as well::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 1, 2]))
      ...                                                  # doctest: +ELLIPSIS
      0.8...
      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      0.66...

    If classes members are completely split across different clusters,
    the assignment is totally incomplete, hence the V-Measure is null::

      >>> print("%.6f" % v_measure_score([0, 0, 0, 0], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    Clusters that include samples from totally different classes totally
    destroy the homogeneity of the labeling, hence::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 0, 0]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred)[2]

def mutual_info_score(labels_true, labels_pred, contingency=None):
    """Mutual Information between two clusterings

    The Mutual Information is a measure of the similarity between two labels of
    the same data. Where :math:`P(i)` is the probability of a random sample
    occurring in cluster :math:`U_i` and :math:`P'(j)` is the probability of a
    random sample occurring in cluster :math:`V_j`, the Mutual Information
    between clusterings :math:`U` and :math:`V` is given as:

    .. math::

        MI(U,V)=\sum_{i=1}^R \sum_{j=1}^C P(i,j)\log\\frac{P(i,j)}{P(i)P'(j)}

    This is equal to the Kullback-Leibler divergence of the joint distribution
    with the product distribution of the marginals.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    contingency: None or array, shape = [n_classes_true, n_classes_pred]
        A contingency matrix given by the :func:`contingency_matrix` function.
        If value is ``None``, it will be computed, otherwise the given value is
        used, with ``labels_true`` and ``labels_pred`` ignored.

    Returns
    -------
    mi: float
       Mutual information, a non-negative value

    See also
    --------
    adjusted_mutual_info_score: Adjusted against chance Mutual Information
    normalized_mutual_info_score: Normalized Mutual Information
    """
    if contingency is None:
        labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
        contingency = contingency_matrix(labels_true, labels_pred)
    contingency = np.array(contingency, dtype='float')
    contingency_sum = np.sum(contingency)
    pi = np.sum(contingency, axis=1)
    pj = np.sum(contingency, axis=0)
    outer = np.outer(pi, pj)
    nnz = contingency != 0.0
    # normalized contingency
    contingency_nm = contingency[nnz]
    log_contingency_nm = np.log(contingency_nm)
    contingency_nm /= contingency_sum
    # log(a / b) should be calculated as log(a) - log(b) for
    # possible loss of precision
    log_outer = -np.log(outer[nnz]) + log(pi.sum()) + log(pj.sum())
    mi = (contingency_nm * (log_contingency_nm - log(contingency_sum))
          + contingency_nm * log_outer)
    return mi.sum()

def adjusted_mutual_info_score(labels_true, labels_pred):
    """Adjusted Mutual Information between two clusterings

    Adjusted Mutual Information (AMI) is an adjustment of the Mutual
    Information (MI) score to account for chance. It accounts for the fact that
    the MI is generally higher for two clusterings with a larger number of
    clusters, regardless of whether there is actually more information shared.
    For two clusterings :math:`U` and :math:`V`, the AMI is given as::

        AMI(U, V) = [MI(U, V) - E(MI(U, V))] / [max(H(U), H(V)) - E(MI(U, V))]

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Be mindful that this function is an order of magnitude slower than other
    metrics, such as the Adjusted Rand Index.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    Returns
    -------
    ami: float(upperlimited by 1.0)
       The AMI returns a value of 1 when the two partitions are identical
       (ie perfectly matched). Random partitions (independent labellings) have
       an expected AMI around 0 on average hence can be negative.

    See also
    --------
    adjusted_rand_score: Adjusted Rand Index
    mutual_information_score: Mutual Information (not adjusted for chance)

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have
    score 1.0::

      >>> from sklearn.metrics.cluster import adjusted_mutual_info_score
      >>> adjusted_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> adjusted_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    If classes members are completely split across different clusters,
    the assignment is totally in-complete, hence the AMI is null::

      >>> adjusted_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    References
    ----------
    .. [1] `Vinh, Epps, and Bailey, (2010). Information Theoretic Measures for
       Clusterings Comparison: Variants, Properties, Normalization and
       Correction for Chance, JMLR
       <http://jmlr.csail.mit.edu/papers/volume11/vinh10a/vinh10a.pdf>`_

    .. [2] `Wikipedia entry for the Adjusted Mutual Information
       <http://en.wikipedia.org/wiki/Adjusted_Mutual_Information>`_

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    n_samples = labels_true.shape[0]
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split.
    # This is a perfect match hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1
            or classes.shape[0] == clusters.shape[0] == 0):
        return 1.0
    contingency = contingency_matrix(labels_true, labels_pred)
    contingency = np.array(contingency, dtype='float')
    # Calculate the MI for the two clusterings
    mi = mutual_info_score(labels_true, labels_pred,
                           contingency=contingency)
    # Calculate the expected value for the mutual information
    emi = expected_mutual_information(contingency, n_samples)
    # Calculate entropy for each labeling
    h_true, h_pred = entropy(labels_true), entropy(labels_pred)
    ami = (mi - emi) / (max(h_true, h_pred) - emi)
    return ami

def normalized_mutual_info_score(labels_true, labels_pred):
    """Normalized Mutual Information between two clusterings

    Normalized Mutual Information (NMI) is an normalization of the Mutual
    Information (MI) score to scale the results between 0 (no mutual
    information) and 1 (perfect correlation). In this function, mutual
    information is normalized by ``sqrt(H(labels_true) * H(labels_pred))``

    This measure is not adjusted for chance. Therefore
    :func:`adjusted_mustual_info_score` might be preferred.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    Returns
    -------
    nmi: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    See also
    --------
    adjusted_rand_score: Adjusted Rand Index
    adjusted_mutual_info_score: Adjusted Mutual Information (adjusted
        against chance)

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have
    score 1.0::

      >>> from sklearn.metrics.cluster import normalized_mutual_info_score
      >>> normalized_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> normalized_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    If classes members are completely split across different clusters,
    the assignment is totally in-complete, hence the NMI is null::

      >>> normalized_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split.
    # This is a perfect match hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1
            or classes.shape[0] == clusters.shape[0] == 0):
        return 1.0
    contingency = contingency_matrix(labels_true, labels_pred)
    contingency = np.array(contingency, dtype='float')
    # Calculate the MI for the two clusterings
    mi = mutual_info_score(labels_true, labels_pred,
                           contingency=contingency)
    # Calculate the expected value for the mutual information
    # Calculate entropy for each labeling
    h_true, h_pred = entropy(labels_true), entropy(labels_pred)
    nmi = mi / max(np.sqrt(h_true * h_pred), 1e-10)
    return nmi

def entropy(labels):
    """Calculates the entropy for a labeling."""
    if len(labels) == 0:
        return 1.0
    label_idx = np.unique(labels, return_inverse=True)[1]
    pi = bincount(label_idx).astype(np.float)
    pi = pi[pi > 0]
    pi_sum = np.sum(pi)
    # log(a / b) should be calculated as log(a) - log(b) for
    # possible loss of precision
    return -np.sum((pi / pi_sum) * (np.log(pi) - log(pi_sum)))


import numpy as np
import pytest

from pandas.compat.numpy import _np_version_under1p16

import pandas as pd
from pandas.core.arrays.numpy_ import PandasArray, PandasDtype
import pandas.util.testing as tm

from . import base

@pytest.fixture(params=["float", "object"])
def dtype(request):
    return PandasDtype(np.dtype(request.param))

@pytest.fixture
def allow_in_pandas(monkeypatch):
    """
    A monkeypatch to tells pandas to let us in.

    By default, passing a PandasArray to an index / series / frame
    constructor will unbox that PandasArray to an ndarray, and treat
    it as a non-EA column. We don't want people using EAs without
    reason.

    The mechanism for this is a check against ABCPandasArray
    in each constructor.

    But, for testing, we need to allow them in pandas. So we patch
    the _typ of PandasArray, so that we evade the ABCPandasArray
    check.
    """
    with monkeypatch.context() as m:
        m.setattr(PandasArray, "_typ", "extension")
        yield

@pytest.fixture
def data(allow_in_pandas, dtype):
    if dtype.numpy_dtype == "object":
        return pd.Series([(i,) for i in range(100)]).array
    return PandasArray(np.arange(1, 101, dtype=dtype._dtype))

@pytest.fixture
def data_missing(allow_in_pandas, dtype):
    # For NumPy <1.16, np.array([np.nan, (1,)]) raises
    # ValueError: setting an array element with a sequence.
    if dtype.numpy_dtype == "object":
        if _np_version_under1p16:
            raise pytest.skip("Skipping for NumPy <1.16")
        return PandasArray(np.array([np.nan, (1,)]))
    return PandasArray(np.array([np.nan, 1.0]))

@pytest.fixture
def na_value():
    return np.nan

@pytest.fixture
def na_cmp():
    def cmp(a, b):
        return np.isnan(a) and np.isnan(b)

    return cmp

@pytest.fixture
def data_for_sorting(allow_in_pandas, dtype):
    """Length-3 array with a known sort order.

    This should be three items [B, C, A] with
    A < B < C
    """
    if dtype.numpy_dtype == "object":
        # Use an empty tuple for first element, then remove,
        # to disable np.array's shape inference.
        return PandasArray(np.array([(), (2,), (3,), (1,)])[1:])
    return PandasArray(np.array([1, 2, 0]))

@pytest.fixture
def data_missing_for_sorting(allow_in_pandas, dtype):
    """Length-3 array with a known sort order.

    This should be three items [B, NA, A] with
    A < B and NA missing.
    """
    if dtype.numpy_dtype == "object":
        return PandasArray(np.array([(1,), np.nan, (0,)]))
    return PandasArray(np.array([1, np.nan, 0]))

@pytest.fixture
def data_for_grouping(allow_in_pandas, dtype):
    """Data for factorization, grouping, and unique tests.

    Expected to be like [B, B, NA, NA, A, A, B, C]

    Where A < B < C and NA is missing
    """
    if dtype.numpy_dtype == "object":
        a, b, c = (1,), (2,), (3,)
    else:
        a, b, c = np.arange(3)
    return PandasArray(np.array([b, b, np.nan, np.nan, a, a, b, c]))

@pytest.fixture
def skip_numpy_object(dtype):
    """
    Tests for PandasArray with nested data. Users typically won't create
    these objects via `pd.array`, but they can show up through `.array`
    on a Series with nested data. Many of the base tests fail, as they aren't
    appropriate for nested data.

    This fixture allows these tests to be skipped when used as a usefixtures
    marker to either an individual test or a test class.
    """
    if dtype == "object":
        raise pytest.skip("Skipping for object dtype.")

skip_nested = pytest.mark.usefixtures("skip_numpy_object")

class BaseNumPyTests:
    pass

class TestCasting(BaseNumPyTests, base.BaseCastingTests):
    @skip_nested
    def test_astype_str(self, data):
        # ValueError: setting an array element with a sequence
        super().test_astype_str(data)

class TestConstructors(BaseNumPyTests, base.BaseConstructorsTests):
    @pytest.mark.skip(reason="We don't register our dtype")
    # We don't want to register. This test should probably be split in two.
    def test_from_dtype(self, data):
        pass

    @skip_nested
    def test_array_from_scalars(self, data):
        # ValueError: PandasArray must be 1-dimensional.
        super().test_array_from_scalars(data)

class TestDtype(BaseNumPyTests, base.BaseDtypeTests):
    @pytest.mark.skip(reason="Incorrect expected.")
    # we unsurprisingly clash with a NumPy name.
    def test_check_dtype(self, data):
        pass

class TestGetitem(BaseNumPyTests, base.BaseGetitemTests):
    @skip_nested
    def test_getitem_scalar(self, data):
        # AssertionError
        super().test_getitem_scalar(data)

    @skip_nested
    def test_take_series(self, data):
        # ValueError: PandasArray must be 1-dimensional.
        super().test_take_series(data)

    @pytest.mark.xfail(reason="astype doesn't recognize data.dtype")
    def test_loc_iloc_frame_single_dtype(self, data):
        super().test_loc_iloc_frame_single_dtype(data)

class TestGroupby(BaseNumPyTests, base.BaseGroupbyTests):
    @skip_nested
    def test_groupby_extension_apply(self, data_for_grouping, groupby_apply_op):
        # ValueError: Names should be list-like for a MultiIndex
        super().test_groupby_extension_apply(data_for_grouping, groupby_apply_op)

class TestInterface(BaseNumPyTests, base.BaseInterfaceTests):
    @skip_nested
    def test_array_interface(self, data):
        # NumPy array shape inference
        super().test_array_interface(data)

class TestMethods(BaseNumPyTests, base.BaseMethodsTests):
    @pytest.mark.skip(reason="TODO: remove?")
    def test_value_counts(self, all_data, dropna):
        pass

    @pytest.mark.skip(reason="Incorrect expected")
    # We have a bool dtype, so the result is an ExtensionArray
    # but expected is not
    def test_combine_le(self, data_repeated):
        super().test_combine_le(data_repeated)

    @skip_nested
    def test_combine_add(self, data_repeated):
        # Not numeric
        super().test_combine_add(data_repeated)

    @skip_nested
    def test_shift_fill_value(self, data):
        # np.array shape inference. Shift implementation fails.
        super().test_shift_fill_value(data)

    @skip_nested
    @pytest.mark.parametrize("box", [pd.Series, lambda x: x])
    @pytest.mark.parametrize("method", [lambda x: x.unique(), pd.unique])
    def test_unique(self, data, box, method):
        # Fails creating expected
        super().test_unique(data, box, method)

    @skip_nested
    def test_fillna_copy_frame(self, data_missing):
        # The "scalar" for this array isn't a scalar.
        super().test_fillna_copy_frame(data_missing)

    @skip_nested
    def test_fillna_copy_series(self, data_missing):
        # The "scalar" for this array isn't a scalar.
        super().test_fillna_copy_series(data_missing)

    @skip_nested
    def test_hash_pandas_object_works(self, data, as_frame):
        # ndarray of tuples not hashable
        super().test_hash_pandas_object_works(data, as_frame)

    @skip_nested
    def test_searchsorted(self, data_for_sorting, as_series):
        # Test setup fails.
        super().test_searchsorted(data_for_sorting, as_series)

    @skip_nested
    def test_where_series(self, data, na_value, as_frame):
        # Test setup fails.
        super().test_where_series(data, na_value, as_frame)

    @skip_nested
    @pytest.mark.parametrize("repeats", [0, 1, 2, [1, 2, 3]])
    def test_repeat(self, data, repeats, as_series, use_numpy):
        # Fails creating expected
        super().test_repeat(data, repeats, as_series, use_numpy)

@skip_nested
class TestArithmetics(BaseNumPyTests, base.BaseArithmeticOpsTests):
    divmod_exc = None
    series_scalar_exc = None
    frame_scalar_exc = None
    series_array_exc = None

    def test_divmod_series_array(self, data):
        s = pd.Series(data)
        self._check_divmod_op(s, divmod, data, exc=None)

    @pytest.mark.skip("We implement ops")
    def test_error(self, data, all_arithmetic_operators):
        pass

    def test_arith_series_with_scalar(self, data, all_arithmetic_operators):
        super().test_arith_series_with_scalar(data, all_arithmetic_operators)

    def test_arith_series_with_array(self, data, all_arithmetic_operators):
        super().test_arith_series_with_array(data, all_arithmetic_operators)

class TestPrinting(BaseNumPyTests, base.BasePrintingTests):
    pass

@skip_nested
class TestNumericReduce(BaseNumPyTests, base.BaseNumericReduceTests):
    def check_reduce(self, s, op_name, skipna):
        result = getattr(s, op_name)(skipna=skipna)
        # avoid coercing int -> float. Just cast to the actual numpy type.
        expected = getattr(s.astype(s.dtype._dtype), op_name)(skipna=skipna)
        tm.assert_almost_equal(result, expected)

@skip_nested
class TestBooleanReduce(BaseNumPyTests, base.BaseBooleanReduceTests):
    pass

class TestMissing(BaseNumPyTests, base.BaseMissingTests):
    @skip_nested
    def test_fillna_scalar(self, data_missing):
        # Non-scalar "scalar" values.
        super().test_fillna_scalar(data_missing)

    @skip_nested
    def test_fillna_series_method(self, data_missing, fillna_method):
        # Non-scalar "scalar" values.
        super().test_fillna_series_method(data_missing, fillna_method)

    @skip_nested
    def test_fillna_series(self, data_missing):
        # Non-scalar "scalar" values.
        super().test_fillna_series(data_missing)

    @skip_nested
    def test_fillna_frame(self, data_missing):
        # Non-scalar "scalar" values.
        super().test_fillna_frame(data_missing)

class TestReshaping(BaseNumPyTests, base.BaseReshapingTests):
    @pytest.mark.skip("Incorrect parent test")
    # not actually a mixed concat, since we concat int and int.
    def test_concat_mixed_dtypes(self, data):
        super().test_concat_mixed_dtypes(data)

    @skip_nested
    def test_merge(self, data, na_value):
        # Fails creating expected
        super().test_merge(data, na_value)

    @skip_nested
    def test_merge_on_extension_array(self, data):
        # Fails creating expected
        super().test_merge_on_extension_array(data)

    @skip_nested
    def test_merge_on_extension_array_duplicates(self, data):
        # Fails creating expected
        super().test_merge_on_extension_array_duplicates(data)

class TestSetitem(BaseNumPyTests, base.BaseSetitemTests):
    @skip_nested
    def test_setitem_scalar_series(self, data, box_in_series):
        # AssertionError
        super().test_setitem_scalar_series(data, box_in_series)

    @skip_nested
    def test_setitem_sequence(self, data, box_in_series):
        # ValueError: shape mismatch: value array of shape (2,1) could not
        # be broadcast to indexing result of shape (2,)
        super().test_setitem_sequence(data, box_in_series)

    @skip_nested
    def test_setitem_sequence_mismatched_length_raises(self, data, as_array):
        # ValueError: PandasArray must be 1-dimensional.
        super().test_setitem_sequence_mismatched_length_raises(data, as_array)

    @skip_nested
    def test_setitem_sequence_broadcasts(self, data, box_in_series):
        # ValueError: cannot set using a list-like indexer with a different
        # length than the value
        super().test_setitem_sequence_broadcasts(data, box_in_series)

    @skip_nested
    def test_setitem_loc_scalar_mixed(self, data):
        # AssertionError
        super().test_setitem_loc_scalar_mixed(data)

    @skip_nested
    def test_setitem_loc_scalar_multiple_homogoneous(self, data):
        # AssertionError
        super().test_setitem_loc_scalar_multiple_homogoneous(data)

    @skip_nested
    def test_setitem_iloc_scalar_mixed(self, data):
        # AssertionError
        super().test_setitem_iloc_scalar_mixed(data)

    @skip_nested
    def test_setitem_iloc_scalar_multiple_homogoneous(self, data):
        # AssertionError
        super().test_setitem_iloc_scalar_multiple_homogoneous(data)

    @skip_nested
    @pytest.mark.parametrize("setter", ["loc", None])
    def test_setitem_mask_broadcast(self, data, setter):
        # ValueError: cannot set using a list-like indexer with a different
        # length than the value
        super().test_setitem_mask_broadcast(data, setter)

    @skip_nested
    def test_setitem_scalar_key_sequence_raise(self, data):
        # Failed: DID NOT RAISE <class 'ValueError'>
        super().test_setitem_scalar_key_sequence_raise(data)

@skip_nested
class TestParsing(BaseNumPyTests, base.BaseParsingTests):
    pass


import ReviewHelper
import pandas as pd

df = ReviewHelper.get_pandas_data_frame_created_from_bibtex_file()

#df_journal = df.groupby('journal')["ID"]
dfJournalList = df.groupby(['journal'])['ID'].count().order(ascending=False)

isOdd = (dfJournalList.size % 2 == 1)
if (isOdd):
    table_row_length = dfJournalList.size / 2 +1
else:
    table_row_length = dfJournalList.size / 2

table_content_inside=""

for index in range(table_row_length):
    journal_name_1column = dfJournalList.index[index]
    journal_count_1column = dfJournalList[index]

    second_column_index = index + table_row_length
    if(second_column_index < dfJournalList.size):
        journal_name_2column = dfJournalList.index[second_column_index]
        journal_count_2column = dfJournalList[second_column_index]
    else:
        journal_name_2column = ""
        journal_count_2column = ""
    line = "{journal_name_1column} & {journal_count_1column} & {journal_name_2column} & {journal_count_2column}   \\\\ \n".format(
        journal_name_1column = journal_name_1column
        ,journal_count_1column = journal_count_1column
        ,journal_name_2column = journal_name_2column
        ,journal_count_2column = journal_count_2column
    )

    table_content_inside = table_content_inside + line

table_content_start = """
\\begin{table*}[!ht]
    \\caption{ \\textbf{Journals and Article Counts} }
    \\label{table-JournalAndArticleCounts}
    \\centering
   \\begin{adjustbox}{max width=\\textwidth}
    \\normalsize
\\begin{tabular}{llll}

\\toprule

Journal Name & Article Count & Journal Name & Article Count \\\\

\\midrule
"""
table_content_end = """

\\bottomrule

\\end{tabular}

\\end{adjustbox}

\\end{table*}
"""

table_content_full = table_content_start + table_content_inside + table_content_end

filename = "../latex/table-JournalAndArticleCounts.tex"
target = open(filename, 'w')
target.write(table_content_full)
target.close()


"""
============================================================
Single estimator versus bagging: bias-variance decomposition
============================================================

This example illustrates and compares the bias-variance decomposition of the
expected mean squared error of a single estimator against a bagging ensemble.

In regression, the expected mean squared error of an estimator can be
decomposed in terms of bias, variance and noise. On average over datasets of
the regression problem, the bias term measures the average amount by which the
predictions of the estimator differ from the predictions of the best possible
estimator for the problem (i.e., the Bayes model). The variance term measures
the variability of the predictions of the estimator when fit over different
instances LS of the problem. Finally, the noise measures the irreducible part
of the error which is due the variability in the data.

The upper left figure illustrates the predictions (in dark red) of a single
decision tree trained over a random dataset LS (the blue dots) of a toy 1d
regression problem. It also illustrates the predictions (in light red) of other
single decision trees trained over other (and different) randomly drawn
instances LS of the problem. Intuitively, the variance term here corresponds to
the width of the beam of predictions (in light red) of the individual
estimators. The larger the variance, the more sensitive are the predictions for
`x` to small changes in the training set. The bias term corresponds to the
difference between the average prediction of the estimator (in cyan) and the
best possible model (in dark blue). On this problem, we can thus observe that
the bias is quite low (both the cyan and the blue curves are close to each
other) while the variance is large (the red beam is rather wide).

The lower left figure plots the pointwise decomposition of the expected mean
squared error of a single decision tree. It confirms that the bias term (in
blue) is low while the variance is large (in green). It also illustrates the
noise part of the error which, as expected, appears to be constant and around
`0.01`.

The right figures correspond to the same plots but using instead a bagging
ensemble of decision trees. In both figures, we can observe that the bias term
is larger than in the previous case. In the upper right figure, the difference
between the average prediction (in cyan) and the best possible model is larger
(e.g., notice the offset around `x=2`). In the lower right figure, the bias
curve is also slightly higher than in the lower left figure. In terms of
variance however, the beam of predictions is narrower, which suggests that the
variance is lower. Indeed, as the lower right figure confirms, the variance
term (in green) is lower than for single decision trees. Overall, the bias-
variance decomposition is therefore no longer the same. The tradeoff is better
for bagging: averaging several decision trees fit on bootstrap copies of the
dataset slightly increases the bias term but allows for a larger reduction of
the variance, which results in a lower overall mean squared error (compare the
red curves int the lower figures). The script output also confirms this
intuition. The total error of the bagging ensemble is lower than the total
error of a single decision tree, and this difference indeed mainly stems from a
reduced variance.

For further details on bias-variance decomposition, see section 7.3 of [1]_.

References
----------

.. [1] T. Hastie, R. Tibshirani and J. Friedman,
       "Elements of Statistical Learning", Springer, 2009.

"""
print(__doc__)

# Author: Gilles Louppe <g.louppe@gmail.com>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import BaggingRegressor
from sklearn.tree import DecisionTreeRegressor

# Settings
n_repeat = 50       # Number of iterations for computing expectations
n_train = 50        # Size of the training set
n_test = 1000       # Size of the test set
noise = 0.1         # Standard deviation of the noise
np.random.seed(0)

# Change this for exploring the bias-variance decomposition of other
# estimators. This should work well for estimators with high variance (e.g.,
# decision trees or KNN), but poorly for estimators with low variance (e.g.,
# linear models).
estimators = [("Tree", DecisionTreeRegressor()),
              ("Bagging(Tree)", BaggingRegressor(DecisionTreeRegressor()))]

n_estimators = len(estimators)

# Generate data
def f(x):
    x = x.ravel()

    return np.exp(-x ** 2) + 1.5 * np.exp(-(x - 2) ** 2)

def generate(n_samples, noise, n_repeat=1):
    X = np.random.rand(n_samples) * 10 - 5
    X = np.sort(X)

    if n_repeat == 1:
        y = f(X) + np.random.normal(0.0, noise, n_samples)
    else:
        y = np.zeros((n_samples, n_repeat))

        for i in range(n_repeat):
            y[:, i] = f(X) + np.random.normal(0.0, noise, n_samples)

    X = X.reshape((n_samples, 1))

    return X, y

X_train = []
y_train = []

for i in range(n_repeat):
    X, y = generate(n_samples=n_train, noise=noise)
    X_train.append(X)
    y_train.append(y)

X_test, y_test = generate(n_samples=n_test, noise=noise, n_repeat=n_repeat)

# Loop over estimators to compare
for n, (name, estimator) in enumerate(estimators):
    # Compute predictions
    y_predict = np.zeros((n_test, n_repeat))

    for i in range(n_repeat):
        estimator.fit(X_train[i], y_train[i])
        y_predict[:, i] = estimator.predict(X_test)

    # Bias^2 + Variance + Noise decomposition of the mean squared error
    y_error = np.zeros(n_test)

    for i in range(n_repeat):
        for j in range(n_repeat):
            y_error += (y_test[:, j] - y_predict[:, i]) ** 2

    y_error /= (n_repeat * n_repeat)

    y_noise = np.var(y_test, axis=1)
    y_bias = (f(X_test) - np.mean(y_predict, axis=1)) ** 2
    y_var = np.var(y_predict, axis=1)

    print("{0}: {1:.4f} (error) = {2:.4f} (bias^2) "
          " + {3:.4f} (var) + {4:.4f} (noise)".format(name,
                                                      np.mean(y_error),
                                                      np.mean(y_bias),
                                                      np.mean(y_var),
                                                      np.mean(y_noise)))

    # Plot figures
    plt.subplot(2, n_estimators, n + 1)
    plt.plot(X_test, f(X_test), "b", label="$f(x)$")
    plt.plot(X_train[0], y_train[0], ".b", label="LS ~ $y = f(x)+noise$")

    for i in range(n_repeat):
        if i == 0:
            plt.plot(X_test, y_predict[:, i], "r", label="$\^y(x)$")
        else:
            plt.plot(X_test, y_predict[:, i], "r", alpha=0.05)

    plt.plot(X_test, np.mean(y_predict, axis=1), "c",
             label="$\mathbb{E}_{LS} \^y(x)$")

    plt.xlim([-5, 5])
    plt.title(name)

    if n == 0:
        plt.legend(loc="upper left", prop={"size": 11})

    plt.subplot(2, n_estimators, n_estimators + n + 1)
    plt.plot(X_test, y_error, "r", label="$error(x)$")
    plt.plot(X_test, y_bias, "b", label="$bias^2(x)$"),
    plt.plot(X_test, y_var, "g", label="$variance(x)$"),
    plt.plot(X_test, y_noise, "c", label="$noise(x)$")

    plt.xlim([-5, 5])
    plt.ylim([0, 0.1])

    if n == 0:
        plt.legend(loc="upper left", prop={"size": 11})

plt.show()


# -*- coding: utf-8 -*-
"""
Tests for Timestamp parsing, aimed at pandas/_libs/tslibs/parsing.pyx
"""
from datetime import datetime

from dateutil.parser import parse
import numpy as np
import pytest

from pandas._libs.tslibs import parsing
from pandas._libs.tslibs.parsing import parse_time_string
import pandas.util._test_decorators as td

from pandas.util import testing as tm

def test_parse_time_string():
    (date, parsed, reso) = parse_time_string("4Q1984")
    (date_lower, parsed_lower, reso_lower) = parse_time_string("4q1984")

    assert date == date_lower
    assert reso == reso_lower
    assert parsed == parsed_lower

@pytest.mark.parametrize("dashed,normal", [
    ("1988-Q2", "1988Q2"),
    ("2Q-1988", "2Q1988")
])
def test_parse_time_quarter_with_dash(dashed, normal):
    # see gh-9688
    (date_dash, parsed_dash, reso_dash) = parse_time_string(dashed)
    (date, parsed, reso) = parse_time_string(normal)

    assert date_dash == date
    assert parsed_dash == parsed
    assert reso_dash == reso

@pytest.mark.parametrize("dashed", [
    "-2Q1992", "2-Q1992", "4-4Q1992"
])
def test_parse_time_quarter_with_dash_error(dashed):
    msg = ("Unknown datetime string format, "
           "unable to parse: {dashed}".format(dashed=dashed))

    with pytest.raises(parsing.DateParseError, match=msg):
        parse_time_string(dashed)

@pytest.mark.parametrize("date_string,expected", [
    ("123.1234", False),
    ("-50000", False),
    ("999", False),
    ("m", False),
    ("T", False),

    ("Mon Sep 16, 2013", True),
    ("2012-01-01", True),
    ("01/01/2012", True),
    ("01012012", True),
    ("0101", True),
    ("1-1", True)
])
def test_does_not_convert_mixed_integer(date_string, expected):
    assert parsing._does_string_look_like_datetime(date_string) is expected

@pytest.mark.parametrize("date_str,kwargs,msg", [
    ("2013Q5", dict(),
     ("Incorrect quarterly string is given, "
      "quarter must be between 1 and 4: 2013Q5")),

    # see gh-5418
    ("2013Q1", dict(freq="INVLD-L-DEC-SAT"),
     ("Unable to retrieve month information "
      "from given freq: INVLD-L-DEC-SAT"))
])
def test_parsers_quarterly_with_freq_error(date_str, kwargs, msg):
    with pytest.raises(parsing.DateParseError, match=msg):
        parsing.parse_time_string(date_str, **kwargs)

@pytest.mark.parametrize("date_str,freq,expected", [
    ("2013Q2", None, datetime(2013, 4, 1)),
    ("2013Q2", "A-APR", datetime(2012, 8, 1)),
    ("2013-Q2", "A-DEC", datetime(2013, 4, 1))
])
def test_parsers_quarterly_with_freq(date_str, freq, expected):
    result, _, _ = parsing.parse_time_string(date_str, freq=freq)
    assert result == expected

@pytest.mark.parametrize("date_str", [
    "2Q 2005", "2Q-200A", "2Q-200",
    "22Q2005", "2Q200.", "6Q-20"
])
def test_parsers_quarter_invalid(date_str):
    if date_str == "6Q-20":
        msg = ("Incorrect quarterly string is given, quarter "
               "must be between 1 and 4: {date_str}".format(date_str=date_str))
    else:
        msg = ("Unknown datetime string format, unable "
               "to parse: {date_str}".format(date_str=date_str))

    with pytest.raises(ValueError, match=msg):
        parsing.parse_time_string(date_str)

@pytest.mark.parametrize("date_str,expected", [
    ("201101", datetime(2011, 1, 1, 0, 0)),
    ("200005", datetime(2000, 5, 1, 0, 0))
])
def test_parsers_month_freq(date_str, expected):
    result, _, _ = parsing.parse_time_string(date_str, freq="M")
    assert result == expected

@td.skip_if_not_us_locale
@pytest.mark.parametrize("string,fmt", [
    ("20111230", "%Y%m%d"),
    ("2011-12-30", "%Y-%m-%d"),
    ("30-12-2011", "%d-%m-%Y"),
    ("2011-12-30 00:00:00", "%Y-%m-%d %H:%M:%S"),
    ("2011-12-30T00:00:00", "%Y-%m-%dT%H:%M:%S"),
    ("2011-12-30 00:00:00.000000", "%Y-%m-%d %H:%M:%S.%f")
])
def test_guess_datetime_format_with_parseable_formats(string, fmt):
    result = parsing._guess_datetime_format(string)
    assert result == fmt

@pytest.mark.parametrize("dayfirst,expected", [
    (True, "%d/%m/%Y"),
    (False, "%m/%d/%Y")
])
def test_guess_datetime_format_with_dayfirst(dayfirst, expected):
    ambiguous_string = "01/01/2011"
    result = parsing._guess_datetime_format(ambiguous_string,
                                            dayfirst=dayfirst)
    assert result == expected

@td.skip_if_has_locale
@pytest.mark.parametrize("string,fmt", [
    ("30/Dec/2011", "%d/%b/%Y"),
    ("30/December/2011", "%d/%B/%Y"),
    ("30/Dec/2011 00:00:00", "%d/%b/%Y %H:%M:%S")
])
def test_guess_datetime_format_with_locale_specific_formats(string, fmt):
    result = parsing._guess_datetime_format(string)
    assert result == fmt

@pytest.mark.parametrize("invalid_dt", [
    "2013", "01/2013", "12:00:00", "1/1/1/1",
    "this_is_not_a_datetime", "51a", 9,
    datetime(2011, 1, 1)
])
def test_guess_datetime_format_invalid_inputs(invalid_dt):
    # A datetime string must include a year, month and a day for it to be
    # guessable, in addition to being a string that looks like a datetime.
    assert parsing._guess_datetime_format(invalid_dt) is None

@pytest.mark.parametrize("string,fmt", [
    ("2011-1-1", "%Y-%m-%d"),
    ("1/1/2011", "%m/%d/%Y"),
    ("30-1-2011", "%d-%m-%Y"),
    ("2011-1-1 0:0:0", "%Y-%m-%d %H:%M:%S"),
    ("2011-1-3T00:00:0", "%Y-%m-%dT%H:%M:%S"),
    ("2011-1-1 00:00:00", "%Y-%m-%d %H:%M:%S")
])
def test_guess_datetime_format_no_padding(string, fmt):
    # see gh-11142
    result = parsing._guess_datetime_format(string)
    assert result == fmt

def test_try_parse_dates():
    arr = np.array(["5/1/2000", "6/1/2000", "7/1/2000"], dtype=object)
    result = parsing.try_parse_dates(arr, dayfirst=True)

    expected = np.array([parse(d, dayfirst=True) for d in arr])
    tm.assert_numpy_array_equal(result, expected)


# The MIT License (MIT)
# Copyright (c) 2018 Massachusetts Institute of Technology
#
# Author: Cody Rude
# This software has been created in projects supported by the US National
# Science Foundation and NASA (PI: Pankratius)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# Standard library imports
from collections import OrderedDict
from io import StringIO

# Scikit Data Access imports
from skdaccess.framework.data_class import DataFetcherStream, TableWrapper

# Third party imports
from six.moves.urllib.parse import urlencode
from six.moves.urllib.request import urlopen
import pandas as pd

class DataFetcher(DataFetcherStream):
    """
    Class for handling data requests to data.lacity.org
    """

    def __init__(self, endpoint, parameters, label, verbose=False, app_token = None, **pandas_kwargs):
        """
        Initialize Data Fetcher for accessing data.lacity.org

        @param endpoint: Data endpoint string
        @param parameters: Parameters to use when retrieving dta
        @param label: Label of pandas dataframe
        @param verbose: Print out extra information
        @param app_token: Application token to use to avoid throttling issues
        @param date_columns
        @param pandas_kwargs: Any additional key word arguments are passed to pandas.read_csv
        """
        self.base_url = 'https://data.lacity.org/resource/'
        self.base_url_and_endpoint = self.base_url + endpoint + '.csv?'

        self.parameters = parameters
        self.label = label
        self.app_token = app_token
        self.pandas_kwargs = pandas_kwargs

        if '$$app_token' in parameters:
            raise RuntimeError("Use app_token option in constructor instead of manually " +
                               "adding it into the the parameters")

        if app_token != None:
            self.parameters['$$app_token'] = app_token

        super(DataFetcher, self).__init__([], verbose)

    def output(self):
        """
        Retrieve data from data.lacity.org

        @return Table wrapper of containing specified data
        """
        data_dict = OrderedDict()
        url_query = self.base_url_and_endpoint + urlencode(self.parameters)

        with urlopen(url_query) as remote_resource:
            raw_string = remote_resource.read().decode()

        string_data = StringIO(raw_string)

        data_dict[self.label] = pd.read_csv(string_data, **self.pandas_kwargs)

        return TableWrapper(data_dict)


import numpy as np
from matplotlib import pyplot as plt
from scipy.misc import toimage
from collections import defaultdict, Counter
from types import SimpleNamespace
from PIL import ImageDraw

# This color table is sourced from https://github.com/trident01/BioExt-1/blob/master/AlignmentImage.java
LIGHT_GRAY = 196
FIXED_COLOR_TABLE = defaultdict(lambda: [0, 0, 0], {
  "A": [255, 0, 0],
  "C": [255, 255, 0],
  "T": [0, 255, 0],
  "G": [190, 0, 95],
  "-": [LIGHT_GRAY, LIGHT_GRAY, LIGHT_GRAY]})
GRAY_GAPS_COLOR_TABLE = defaultdict(lambda: [0, 0, 0], {
  "-": [LIGHT_GRAY, LIGHT_GRAY, LIGHT_GRAY]})
BLACK_COLOR_TABLE = defaultdict(lambda: [0, 0, 0])

class Alignment:
  def __init__(self, query_start, query_seq, target_start, target_seq, sequence_name, target_label, expected_errors):
    self.name = sequence_name
    self.target_label = target_label
    self.expected_errors = expected_errors
    self.query_start = int(query_start) - 1
    self.query_seq = query_seq
    query_gap_count = query_seq.count("-")
    self.query_length = len(query_seq) - query_gap_count
    self.target_start = int(target_start) - 1
    self.target_seq = target_seq
    target_gap_count = target_seq.count("-")
    self.target_length = len(target_seq) - target_gap_count
    self.no_gap_length = len(target_seq) - target_gap_count - query_gap_count
    if len(target_seq) != len(query_seq):
      raise ValueError("Length of target sequence not equal to length of query sequence")

def alignment_iterator(alignment, ignore_case=True, include_gaps=False):
  target_index = 0
  target_offset = 0
  query_index = 0
  while target_index < len(alignment.target_seq) and query_index < len(alignment.query_seq):
    if alignment.target_seq[target_index] == "-": # If it is an insertion
      target_offset += 1
    elif alignment.query_seq[query_index] != "-" or include_gaps:
      reference_index = alignment.target_start + target_index - target_offset
      query_nucleotide = alignment.query_seq[query_index].upper() if ignore_case else alignment.query_seq[query_index]
      target_nucleotide = alignment.target_seq[target_index].upper() if ignore_case else alignment.target_seq[target_index]
      yield SimpleNamespace(reference_index=reference_index,
                            target_nucleotide=target_nucleotide,
                            query_nucleotide=query_nucleotide)
    target_index += 1
    query_index += 1

def count_mismatches(alignment, ignore_case=True):
  mismatch_count = 0
  for position in alignment_iterator(alignment, ignore_case):
    if position.target_nucleotide != position.query_nucleotide:
      mismatch_count += 1
  return mismatch_count

def save_expected_error_rates(alignments, output_file):
  expected_error_rates = [a.expected_errors / a.query_length for a in alignments]

  plt.cla()
  plt.hist(expected_error_rates, 50, log=True)
  plt.ylim(ymin=0.9)
  plt.xlabel('Expected Error Rate')
  plt.ylabel('Number of sequences')
  plt.tick_params(which='both', direction='out')
  plt.title('Expected Error Rates')
  plt.grid(True)
  plt.savefig(output_file)

def save_mismatch_rates(alignments, output_file, ignore_case=True):
  mismatch_rates = [count_mismatches(a, ignore_case) / a.no_gap_length for a in alignments]

  plt.cla()
  plt.hist(mismatch_rates, 50, log=True)
  plt.ylim(ymin=0.9)
  plt.xlabel('Rate of mismatches')
  plt.ylabel('Number of sequences')
  plt.tick_params(which='both', direction='out')
  plt.title('Mismatch Rates')
  plt.grid(True)
  plt.savefig(output_file)

def gap_distribution(sequence):
  dist = Counter()
  count_length = 0
  for char in sequence:
    if char == "-":
      count_length += 1
    elif count_length > 0:
      dist[count_length] += 1
      count_length = 0
  if count_length > 0:
    dist[count_length] += 1
  return dist

def save_insertion_or_deletion_dist(alignments, output_file, insertion_not_deletion=True):
  size_counter = Counter()
  for a in alignments:
    size_counter += gap_distribution(a.target_seq if insertion_not_deletion else a.query_seq)

  sizes, counts = zip(*size_counter.items())
  number_of_bins = max(sizes)
  number_of_bins = round(number_of_bins / np.ceil(number_of_bins/50))
  plt.cla()
  n, bins, patches = plt.hist(sizes, number_of_bins, weights=counts, log=True)
  plt.ylim(ymin=0.9)
  plt.xlim(xmin=1)
  plt.xlabel('Size of insertion' if insertion_not_deletion else 'Size of deletion')
  plt.ylabel('Count')
  plt.tick_params(which='both', direction='out')
  plt.title('Insertion size distribution' if insertion_not_deletion else 'Deletion size distribution')
  plt.grid(True)
  plt.savefig(output_file)

# Get nucleotide distribution
def nucleotide_distribution(alignments, ignore_case=False, include_gaps=True):
  max_index = 0
  distribution = defaultdict(Counter)
  for a in alignments:
    for position in alignment_iterator(a, ignore_case, include_gaps):
      distribution[position.reference_index][position.query_nucleotide] += 1
    max_index = max(max_index, a.target_start + a.target_length)
  return [distribution[i] for i in range(max_index)]

def save_nucleotide_map(alignments, output, ignore_case=True, include_gaps=True):
  nucleotides = nucleotide_distribution(alignments, ignore_case, include_gaps)
  width = len(nucleotides)
  keys = set()
  for distribution_at_base in nucleotides:
    keys.update(set(distribution_at_base.keys()))
  keys = sorted(list(keys), key=lambda x: "ZZZ" if x == "-" else x)
  nucleotide_count_array = np.zeros((len(keys), width), dtype=np.uint32)
  for i, key in enumerate(keys):
    for j, counts in enumerate(nucleotides):
      nucleotide_count_array[i, j] = counts[key]
  cum_sum = nucleotide_count_array.cumsum(axis=0)
  height = cum_sum[-1,].max()
  data_matrix = np.full((height, width, 3), 255, dtype=np.uint8)
  for x in range(width):
    for i, key in enumerate(keys):
      start = 0 if i == 0 else cum_sum[i - 1, x]
      end = cum_sum[i, x]
      data_matrix[start:end, x, 0:3] = FIXED_COLOR_TABLE[key]
  img = to_image(data_matrix[::-1,], ruler_underneath=True)
  img.save(output)

# Get coverage map
def coverage_map(alignments, include_gaps=False):
  max_index = 0
  coverage = Counter()
  for a in alignments:
    for position in alignment_iterator(a, True, include_gaps):
      coverage[position.reference_index] += 1
    max_index = max(max_index, a.target_start + a.target_length)
  return [coverage[i] for i in range(max_index)]

def save_coverage_map(alignments, output):
  coverage_with_gaps = coverage_map(alignments, True)
  coverage_without_gaps = coverage_map(alignments, False)
  width = len(coverage_with_gaps)
  height = max(coverage_with_gaps)
  data_matrix = np.full((height, width, 3), 255, dtype=np.uint8)
  for x in range(width):
    y1 = coverage_without_gaps[x]
    y2 = coverage_with_gaps[x]
    data_matrix[0:y1, x, 0:3] = 0
    data_matrix[y1:y2, x, 0:3] = 127
  img = to_image(data_matrix[::-1], add_ruler=True, ruler_underneath=True)
  img.save(output)

def save_alignment_map(coords, output_file, sort_key=sum, crop=True, no_ruler=False):
  if crop:
    minimum = min(coords, key=lambda x: x[0])[0]
  else:
    minimum = 0
  maximum = max(coords, key=lambda x: x[1])[1]
  dimensions = (len(coords), maximum - minimum)
  data_matrix = np.full((dimensions[0], dimensions[1] + 1), 255, dtype=np.uint8)
  if sort_key is not None:
    coords.sort(key=sort_key)

  is_multiple_alignment = len(coords[0]) > 3 and type(coords[0][3]) == list

  # Greyscale over the bounds (or black if not multiple alignment)
  for i, coord in enumerate(coords):
    start = coord[0]
    end = coord[1]
    # np.put(data_matrix[i], range(start - minimum, end - minimum), 0)
    data_matrix[i, (start - minimum):(end - minimum)] = LIGHT_GRAY if is_multiple_alignment else 0

  # Black over the subalignments, if any
  if is_multiple_alignment:
    for i, coord in enumerate(coords):
      for subalignment in coord[3]:
        start = subalignment[0]
        end = subalignment[1]
        # np.put(data_matrix[i], range(start - minimum, end - minimum), 0)
        data_matrix[i, (start - minimum):(end - minimum)] = 0

  img = to_image(data_matrix, not no_ruler, offset=minimum)
  img.save(output_file)

def to_image(data_matrix, add_ruler=True, ruler_underneath = False, offset=1):
  maximum = offset + data_matrix.shape[1]
  if add_ruler:
    shape = list(data_matrix.shape)
    shape[0] = 12 # Number of rows
    ruler_matrix = np.full(shape, 255, dtype=data_matrix.dtype)
    # tens ticks
    ruler_matrix[0 if ruler_underneath else 11, 10-(offset%10)::10] = 0
    # 50s ticks
    ruler_matrix[1 if ruler_underneath else 10, 50-(offset%50)::50] = 0
    if ruler_underneath:
      img = toimage(np.vstack([data_matrix, ruler_matrix]))
    else:
      img = toimage(np.vstack([ruler_matrix, data_matrix]))
    draw = ImageDraw.Draw(img)
    # Hundreds words
    for i in range((offset//100) + 1, maximum // 100 + 1):
      centering = (6 * (int(np.log10(i)) + 3) - 1) // 2
      draw.text((i * 100 - centering - offset, (data_matrix.shape[0] + 2) if ruler_underneath else 0), str(i) + "00", fill="black")
  else:
    img = toimage(data_matrix)
  return img


# -*- coding: utf-8 -*-
#
#  2015-07-16 Initial writeup
#  (c) Cornelius Kölbel
#  License:  AGPLv3
#  contact:  http://www.privacyidea.org
#
# This code is free software; you can redistribute it and/or
# modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
# License as published by the Free Software Foundation; either
# version 3 of the License, or any later version.
#
# This code is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
__doc__ = """This module reads audit data and can create statistics from
audit data using pandas.

This module is tested in tests/test_lib_stats.py
"""
import logging
from privacyidea.lib.log import log_with
import datetime
import StringIO
log = logging.getLogger(__name__)

try:
    import matplotlib
    MATPLOT_READY = True
    matplotlib.style.use('ggplot')
    matplotlib.use('Agg')
except Exception as exx:
    MATPLOT_READY = False
    log.warning("If you want to see statistics you need to install python "
                "matplotlib.")

customcmap = [(1, 0, 0), (0, 1, 0), (0, 0, 1)]

@log_with(log)
def get_statistics(auditobject, start_time=datetime.datetime.now()
                                         -datetime.timedelta(days=7),
                   end_time=datetime.datetime.now()):
    """
    Create audit statistics and return a JSON object
    The auditobject is passed from the upper level, usually from the REST API
    as g.auditobject.

    :param auditobject: The audit object
    :type auditobject: Audit Object as defined in auditmodules.base.Audit
    :return: JSON
    """
    result = {}
    df = auditobject.get_dataframe(start_time=start_time, end_time=end_time)

    # authentication successful/fail per user or serial
    for key in ["user", "serial"]:
        result["validate_%s_plot" % key] = _get_success_fail(df, key)

    # get simple usage
    for key in ["serial", "action"]:
        result["%s_plot" % key] = _get_number_of(df, key)

    # failed authentication requests
    for key in ["user", "serial"]:
        result["validate_failed_%s_plot" % key] = _get_fail(df, key)

    result["admin_plot"] = _get_number_of(df, "action", nums=20)

    return result

def _get_success_fail(df, key):

    try:
        output = StringIO.StringIO()
        series = df[df.action.isin(["POST /validate/check",
                                    "GET /validate/check"])].groupby([key,
                                                                'success']).size().unstack()
        fig = series.plot(kind="bar", stacked=True,
                          legend=True,
                          title="Authentications",
                          grid=True,
                          color=customcmap).get_figure()
        fig.savefig(output, format="png")
        o_data = output.getvalue()
        output.close()
        image_data = o_data.encode("base64")
        image_uri = 'data:image/png;base64,%s' % image_data
    except Exception as exx:
        log.info(exx)
        image_uri = "%s" % exx
    return image_uri

def _get_fail(df, key):

    try:
        output = StringIO.StringIO()
        series = df[(df.success==0)
                    & (df.action.isin(["POST /validate/check",
                                       "GET /validate/check"]))][
                     key].value_counts()[:5]

        plot_canvas = matplotlib.pyplot.figure()
        ax = plot_canvas.add_subplot(1,1,1)

        fig = series.plot(ax=ax, kind="bar",
                          colormap="Reds",
                          stacked=False,
                          legend=False,
                          grid=True,
                          title="Failed Authentications").get_figure()
        fig.savefig(output, format="png")
        o_data = output.getvalue()
        output.close()
        image_data = o_data.encode("base64")
        image_uri = 'data:image/png;base64,%s' % image_data
    except Exception as exx:
        log.info(exx)
        image_uri = "%s" % exx
    return image_uri

def _get_number_of(df, key, nums=5):
    """
    return a data url image with a single keyed value.
    It plots the "nums" most occurrences of the "key" column in the dataframe.

    :param df: The DataFrame
    :type df: Pandas DataFrame
    :param key: The key, which should be plotted.
    :param count: how many of the most often values should be plotted
    :return: A data url
    """
    output = StringIO.StringIO()
    output.truncate(0)
    try:
        plot_canvas = matplotlib.pyplot.figure()
        ax = plot_canvas.add_subplot(1, 1, 1)

        series = df[key].value_counts()[:nums]
        fig = series.plot(ax=ax, kind="bar", colormap="Blues",
                          legend=False,
                          stacked=False,
                          title="Numbers of %s" % key,
                          grid=True).get_figure()
        fig.savefig(output, format="png")
        o_data = output.getvalue()
        output.close()
        image_data = o_data.encode("base64")
        image_uri = 'data:image/png;base64,%s' % image_data
    except Exception as exx:
        log.info(exx)
        image_uri = "No data"
    return image_uri