"""
The :mod:`sklearn.kernel_approximation` module implements several
approximate kernel feature maps base on Fourier transforms.
"""

# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#
# License: BSD 3 clause

import warnings

import numpy as np
import scipy.sparse as sp
from scipy.linalg import svd

from .base import BaseEstimator
from .base import TransformerMixin
from .utils import check_array, check_random_state, as_float_array
from .utils.extmath import safe_sparse_dot
from .utils.validation import check_is_fitted
from .metrics.pairwise import pairwise_kernels

class RBFSampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of an RBF kernel by Monte Carlo approximation
    of its Fourier transform.

    It implements a variant of Random Kitchen Sinks.[1]

    Read more in the :ref:`User Guide <rbf_kernel_approx>`.

    Parameters
    ----------
    gamma : float
        Parameter of RBF kernel: exp(-gamma * x^2)

    n_components : int
        Number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    Notes
    -----
    See "Random Features for Large-Scale Kernel Machines" by A. Rahimi and
    Benjamin Recht.

    [1] "Weighted Sums of Random Kitchen Sinks: Replacing
    minimization with randomization in learning" by A. Rahimi and
    Benjamin Recht.
    (http://www.eecs.berkeley.edu/~brecht/papers/08.rah.rec.nips.pdf)
    """

    def __init__(self, gamma=1., n_components=100, random_state=None):
        self.gamma = gamma
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = check_array(X, accept_sparse='csr')
        random_state = check_random_state(self.random_state)
        n_features = X.shape[1]

        self.random_weights_ = (np.sqrt(2 * self.gamma) * random_state.normal(
            size=(n_features, self.n_components)))

        self.random_offset_ = random_state.uniform(0, 2 * np.pi,
                                                   size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
        """
        check_is_fitted(self, 'random_weights_')

        X = check_array(X, accept_sparse='csr')
        projection = safe_sparse_dot(X, self.random_weights_)
        projection += self.random_offset_
        np.cos(projection, projection)
        projection *= np.sqrt(2.) / np.sqrt(self.n_components)
        return projection

class SkewedChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of the "skewed chi-squared" kernel by Monte
    Carlo approximation of its Fourier transform.

    Read more in the :ref:`User Guide <skewed_chi_kernel_approx>`.

    Parameters
    ----------
    skewedness : float
        "skewedness" parameter of the kernel. Needs to be cross-validated.

    n_components : int
        number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    References
    ----------
    See "Random Fourier Approximations for Skewed Multiplicative Histogram
    Kernels" by Fuxin Li, Catalin Ionescu and Cristian Sminchisescu.

    See also
    --------
    AdditiveChi2Sampler : A different approach for approximating an additive
        variant of the chi squared kernel.

    sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel.
    """

    def __init__(self, skewedness=1., n_components=100, random_state=None):
        self.skewedness = skewedness
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = check_array(X)
        random_state = check_random_state(self.random_state)
        n_features = X.shape[1]
        uniform = random_state.uniform(size=(n_features, self.n_components))
        # transform by inverse CDF of sech
        self.random_weights_ = (1. / np.pi
                                * np.log(np.tan(np.pi / 2. * uniform)))
        self.random_offset_ = random_state.uniform(0, 2 * np.pi,
                                                   size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
        """
        check_is_fitted(self, 'random_weights_')

        X = as_float_array(X, copy=True)
        X = check_array(X, copy=False)
        if (X < 0).any():
            raise ValueError("X may not contain entries smaller than zero.")

        X += self.skewedness
        np.log(X, X)
        projection = safe_sparse_dot(X, self.random_weights_)
        projection += self.random_offset_
        np.cos(projection, projection)
        projection *= np.sqrt(2.) / np.sqrt(self.n_components)
        return projection

class AdditiveChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximate feature map for additive chi2 kernel.

    Uses sampling the fourier transform of the kernel characteristic
    at regular intervals.

    Since the kernel that is to be approximated is additive, the components of
    the input vectors can be treated separately.  Each entry in the original
    space is transformed into 2*sample_steps+1 features, where sample_steps is
    a parameter of the method. Typical values of sample_steps include 1, 2 and
    3.

    Optimal choices for the sampling interval for certain data ranges can be
    computed (see the reference). The default values should be reasonable.

    Read more in the :ref:`User Guide <additive_chi_kernel_approx>`.

    Parameters
    ----------
    sample_steps : int, optional
        Gives the number of (complex) sampling points.
    sample_interval : float, optional
        Sampling interval. Must be specified when sample_steps not in {1,2,3}.

    Notes
    -----
    This estimator approximates a slightly different version of the additive
    chi squared kernel then ``metric.additive_chi2`` computes.

    See also
    --------
    SkewedChi2Sampler : A Fourier-approximation to a non-additive variant of
        the chi squared kernel.

    sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel.

    sklearn.metrics.pairwise.additive_chi2_kernel : The exact additive chi
        squared kernel.

    References
    ----------
    See `"Efficient additive kernels via explicit feature maps"
    <http://www.robots.ox.ac.uk/~vedaldi/assets/pubs/vedaldi11efficient.pdf>`_
    A. Vedaldi and A. Zisserman, Pattern Analysis and Machine Intelligence,
    2011
    """

    def __init__(self, sample_steps=2, sample_interval=None):
        self.sample_steps = sample_steps
        self.sample_interval = sample_interval

    def fit(self, X, y=None):
        """Set parameters."""
        X = check_array(X, accept_sparse='csr')
        if self.sample_interval is None:
            # See reference, figure 2 c)
            if self.sample_steps == 1:
                self.sample_interval_ = 0.8
            elif self.sample_steps == 2:
                self.sample_interval_ = 0.5
            elif self.sample_steps == 3:
                self.sample_interval_ = 0.4
            else:
                raise ValueError("If sample_steps is not in [1, 2, 3],"
                                 " you need to provide sample_interval")
        else:
            self.sample_interval_ = self.sample_interval
        return self

    def transform(self, X, y=None):
        """Apply approximate feature map to X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = (n_samples, n_features)

        Returns
        -------
        X_new : {array, sparse matrix}, \
               shape = (n_samples, n_features * (2*sample_steps + 1))
            Whether the return value is an array of sparse matrix depends on
            the type of the input X.
        """
        msg = ("%(name)s is not fitted. Call fit to set the parameters before"
               " calling transform")
        check_is_fitted(self, "sample_interval_", msg=msg)

        X = check_array(X, accept_sparse='csr')
        sparse = sp.issparse(X)

        # check if X has negative values. Doesn't play well with np.log.
        if ((X.data if sparse else X) < 0).any():
            raise ValueError("Entries of X must be non-negative.")
        # zeroth component
        # 1/cosh = sech
        # cosh(0) = 1.0

        transf = self._transform_sparse if sparse else self._transform_dense
        return transf(X)

    def _transform_dense(self, X):
        non_zero = (X != 0.0)
        X_nz = X[non_zero]

        X_step = np.zeros_like(X)
        X_step[non_zero] = np.sqrt(X_nz * self.sample_interval_)

        X_new = [X_step]

        log_step_nz = self.sample_interval_ * np.log(X_nz)
        step_nz = 2 * X_nz * self.sample_interval_

        for j in range(1, self.sample_steps):
            factor_nz = np.sqrt(step_nz /
                                np.cosh(np.pi * j * self.sample_interval_))

            X_step = np.zeros_like(X)
            X_step[non_zero] = factor_nz * np.cos(j * log_step_nz)
            X_new.append(X_step)

            X_step = np.zeros_like(X)
            X_step[non_zero] = factor_nz * np.sin(j * log_step_nz)
            X_new.append(X_step)

        return np.hstack(X_new)

    def _transform_sparse(self, X):
        indices = X.indices.copy()
        indptr = X.indptr.copy()

        data_step = np.sqrt(X.data * self.sample_interval_)
        X_step = sp.csr_matrix((data_step, indices, indptr),
                               shape=X.shape, dtype=X.dtype, copy=False)
        X_new = [X_step]

        log_step_nz = self.sample_interval_ * np.log(X.data)
        step_nz = 2 * X.data * self.sample_interval_

        for j in range(1, self.sample_steps):
            factor_nz = np.sqrt(step_nz /
                                np.cosh(np.pi * j * self.sample_interval_))

            data_step = factor_nz * np.cos(j * log_step_nz)
            X_step = sp.csr_matrix((data_step, indices, indptr),
                                   shape=X.shape, dtype=X.dtype, copy=False)
            X_new.append(X_step)

            data_step = factor_nz * np.sin(j * log_step_nz)
            X_step = sp.csr_matrix((data_step, indices, indptr),
                                   shape=X.shape, dtype=X.dtype, copy=False)
            X_new.append(X_step)

        return sp.hstack(X_new)

class Nystroem(BaseEstimator, TransformerMixin):
    """Approximate a kernel map using a subset of the training data.

    Constructs an approximate feature map for an arbitrary kernel
    using a subset of the data as basis.

    Read more in the :ref:`User Guide <nystroem_kernel_approx>`.

    Parameters
    ----------
    kernel : string or callable, default="rbf"
        Kernel map to be approximated. A callable should accept two arguments
        and the keyword arguments passed to this object as kernel_params, and
        should return a floating point number.

    n_components : int
        Number of features to construct.
        How many data points will be used to construct the mapping.

    gamma : float, default=None
        Gamma parameter for the RBF, polynomial, exponential chi2 and
        sigmoid kernels. Interpretation of the default value is left to
        the kernel; see the documentation for sklearn.metrics.pairwise.
        Ignored by other kernels.

    degree : float, default=3
        Degree of the polynomial kernel. Ignored by other kernels.

    coef0 : float, default=1
        Zero coefficient for polynomial and sigmoid kernels.
        Ignored by other kernels.

    kernel_params : mapping of string to any, optional
        Additional parameters (keyword arguments) for kernel function passed
        as callable object.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    Attributes
    ----------
    components_ : array, shape (n_components, n_features)
        Subset of training points used to construct the feature map.

    component_indices_ : array, shape (n_components)
        Indices of ``components_`` in the training set.

    normalization_ : array, shape (n_components, n_components)
        Normalization matrix needed for embedding.
        Square root of the kernel matrix on ``components_``.

    References
    ----------
    * Williams, C.K.I. and Seeger, M.
      "Using the Nystroem method to speed up kernel machines",
      Advances in neural information processing systems 2001

    * T. Yang, Y. Li, M. Mahdavi, R. Jin and Z. Zhou
      "Nystroem Method vs Random Fourier Features: A Theoretical and Empirical
      Comparison",
      Advances in Neural Information Processing Systems 2012

    See also
    --------
    RBFSampler : An approximation to the RBF kernel using random Fourier
                 features.

    sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels.
    """
    def __init__(self, kernel="rbf", gamma=None, coef0=1, degree=3,
                 kernel_params=None, n_components=100, random_state=None):
        self.kernel = kernel
        self.gamma = gamma
        self.coef0 = coef0
        self.degree = degree
        self.kernel_params = kernel_params
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit estimator to data.

        Samples a subset of training points, computes kernel
        on these and computes normalization matrix.

        Parameters
        ----------
        X : array-like, shape=(n_samples, n_feature)
            Training data.
        """
        X = check_array(X, accept_sparse='csr')
        rnd = check_random_state(self.random_state)
        n_samples = X.shape[0]

        # get basis vectors
        if self.n_components > n_samples:
            # XXX should we just bail?
            n_components = n_samples
            warnings.warn("n_components > n_samples. This is not possible.\n"
                          "n_components was set to n_samples, which results"
                          " in inefficient evaluation of the full kernel.")

        else:
            n_components = self.n_components
        n_components = min(n_samples, n_components)
        inds = rnd.permutation(n_samples)
        basis_inds = inds[:n_components]
        basis = X[basis_inds]

        basis_kernel = pairwise_kernels(basis, metric=self.kernel,
                                        filter_params=True,
                                        **self._get_kernel_params())

        # sqrt of kernel matrix on basis vectors
        U, S, V = svd(basis_kernel)
        S = np.maximum(S, 1e-12)
        self.normalization_ = np.dot(U * 1. / np.sqrt(S), V)
        self.components_ = basis
        self.component_indices_ = inds
        return self

    def transform(self, X):
        """Apply feature map to X.

        Computes an approximate feature map using the kernel
        between some training points and X.

        Parameters
        ----------
        X : array-like, shape=(n_samples, n_features)
            Data to transform.

        Returns
        -------
        X_transformed : array, shape=(n_samples, n_components)
            Transformed data.
        """
        check_is_fitted(self, 'components_')
        X = check_array(X, accept_sparse='csr')

        kernel_params = self._get_kernel_params()
        embedded = pairwise_kernels(X, self.components_,
                                    metric=self.kernel,
                                    filter_params=True,
                                    **kernel_params)
        return np.dot(embedded, self.normalization_.T)

    def _get_kernel_params(self):
        params = self.kernel_params
        if params is None:
            params = {}
        if not callable(self.kernel):
            params['gamma'] = self.gamma
            params['degree'] = self.degree
            params['coef0'] = self.coef0

        return params


"""
Tests for statistics for gravity-style spatial interaction models
"""

__author__ = 'toshan'

import unittest
import numpy as np
import pandas as pd
import gravity as grav
import mle_stats as stats

class SingleParameter(unittest.TestCase):
    """Unit tests statistics when there is a single parameters estimated"""
    def setUp(self):
        self.f = np.array([0, 6469, 7629, 20036, 4690,
                           6194, 11688, 2243, 8857, 7248,
                           3559, 9221, 10099, 22866, 3388,
                           9986, 46618, 11639, 1380, 5261,
                           5985, 6731, 2704, 12250, 16132])
        self.o = np.repeat(1, 25)
        self.d = np.array(range(1, 26))
        self.dij = np.array([0, 576, 946, 597, 373,
                             559, 707, 1208, 602, 692,
                             681, 1934, 332, 595, 906,
                             425, 755, 672, 1587, 526,
                             484, 2141, 2182, 410, 540])
        self.pop = np.array([1596000, 2071000, 3376000, 6978000, 1345000,
                             2064000, 2378000, 1239000, 4435000, 1999000,
                             1274000, 7042000, 834000, 1268000, 1965000,
                             1046000, 12131000, 4824000, 969000, 2401000,
                             2410000, 2847000, 1425000, 1089000, 2909000])
        self.dt = pd.DataFrame({'origins': self.o,
                                'destinations': self.d,
                                'pop': self.pop,
                                'Dij': self.dij,
                                'flows': self.f})
    def test_single_parameter(self):
        model = grav.ProductionConstrained(self.dt, 'origins', 'destinations', 'flows',
            ['pop'], 'Dij', 'pow')
        ss = {'obs_mean_trip_len': 736.52834197296534,
              'pred_mean_trip_len': 734.40974204773784,
              'OD_pairs': 24,
              'predicted_flows': 242873.00000000003,
              'avg_dist_trav': 737.0,
              'num_destinations': 24,
              'observed_flows': 242873,
              'avg_dist': 851.0,
              'num_origins': 1}
        ps = {'beta': {'LL_zero_val': -3.057415839736517,
                       'relative_likelihood_stat': 24833.721614296166,
                       'standard_error': 0.0052734418614330883},
              'all_params': {'zero_vals_LL': -3.1780538303479453,
                             'mle_vals_LL': -3.0062909275101761},
              'pop': {'LL_zero_val': -3.1773474269437778,
                      'relative_likelihood_stat': 83090.010373874276,
                      'standard_error': 0.0027673052892085684}}
        fs = {'r_squared': 0.60516003720997413,
              'srmse': 0.57873206718148507}
        es = {'pred_obs_deviance': 0.1327,
              'entropy_ratio': 0.5642,
              'maximum_entropy': 3.1781,
              'max_pred_deviance': 0.1718,
              'variance_obs_entropy': 2.55421e-06,
              'predicted_entropy': 3.0063,
              't_stat_entropy': 66.7614,
              'max_obs_deviance': 0.3045,
              'observed_entropy': 2.8736,
              'variance_pred_entropy': 1.39664e-06}
        sys_stats = stats.sys_stats(model)
        self.assertAlmostEqual(model.system_stats['obs_mean_trip_len'], ss['obs_mean_trip_len'], 4)
        self.assertAlmostEqual(model.system_stats['pred_mean_trip_len'], ss['pred_mean_trip_len'], 4)
        self.assertAlmostEqual(model.system_stats['OD_pairs'], ss['OD_pairs'])
        self.assertAlmostEqual(model.system_stats['predicted_flows'], ss['predicted_flows'])
        self.assertAlmostEqual(model.system_stats['avg_dist_trav'], ss['avg_dist_trav'])
        self.assertAlmostEqual(model.system_stats['num_destinations'], ss['num_destinations'])
        self.assertAlmostEqual(model.system_stats['observed_flows'], ss['observed_flows'])
        self.assertAlmostEqual(model.system_stats['avg_dist'], ss['avg_dist'], 4)
        self.assertAlmostEqual(model.system_stats['num_origins'], ss['num_origins'])
        param_stats = stats.param_stats(model)
        self.assertAlmostEqual(model.parameter_stats['beta']['LL_zero_val'], ps['beta']['LL_zero_val'], 4)
        self.assertAlmostEqual(model.parameter_stats['beta']['relative_likelihood_stat'],
                                                  ps['beta']['relative_likelihood_stat'], 4)
        self.assertAlmostEqual(model.parameter_stats['beta']['standard_error'], ps['beta']['standard_error'], 4)
        self.assertAlmostEqual(model.parameter_stats['pop']['LL_zero_val'], ps['pop']['LL_zero_val'], 4)
        self.assertAlmostEqual(model.parameter_stats['pop']['relative_likelihood_stat'],
                                                  ps['pop']['relative_likelihood_stat'], 4)
        self.assertAlmostEqual(model.parameter_stats['pop']['standard_error'], ps['pop']['standard_error'], 4)
        self.assertAlmostEqual(model.parameter_stats['all_params']['zero_vals_LL'], ps['all_params']['zero_vals_LL'], 4)
        self.assertAlmostEqual(model.parameter_stats['all_params']['mle_vals_LL'], ps['all_params']['mle_vals_LL'], 4)
        fit_stats = stats.fit_stats(model)
        self.assertAlmostEqual(model.fit_stats['r_squared'], fs['r_squared'], 4)
        self.assertAlmostEqual(model.fit_stats['srmse'], fs['srmse'], 4)
        ent_stats = stats.ent_stats(model)
        self.assertAlmostEqual(model.entropy_stats['pred_obs_deviance'], es['pred_obs_deviance'], 4)
        self.assertAlmostEqual(model.entropy_stats['entropy_ratio'], es['entropy_ratio'], 4)
        self.assertAlmostEqual(model.entropy_stats['maximum_entropy'], es['maximum_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['max_pred_deviance'], es['max_pred_deviance'], 4)
        self.assertAlmostEqual(model.entropy_stats['variance_obs_entropy'], es['variance_obs_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['predicted_entropy'], es['predicted_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['t_stat_entropy'], es['t_stat_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['max_obs_deviance'], es['max_obs_deviance'], 4)
        self.assertAlmostEqual(model.entropy_stats['observed_entropy'], es['observed_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['variance_pred_entropy'], es['variance_pred_entropy'], 4)

class MultipleParameter(unittest.TestCase):
    """Unit tests statistics when there are multiple parameters estimated"""
    def setUp(self):
        self.f = np.array([0, 180048, 79223, 26887, 198144, 17995, 35563, 30528, 110792,
                        283049, 0, 300345, 67280, 718673, 55094, 93434, 87987, 268458,
                        87267, 237229, 0, 281791, 551483, 230788, 178517, 172711, 394481,
                        29877, 60681, 286580, 0, 143860, 49892, 185618, 181868, 274629,
                        130830, 382565, 346407, 92308, 0, 252189, 192223, 89389, 279739,
                        21434, 53772, 287340, 49828, 316650, 0, 141679, 27409, 87938,
                        30287, 64645, 161645, 144980, 199466, 121366, 0, 134229, 289880,
                        21450, 43749, 97808, 113683, 89806, 25574, 158006, 0, 437255,
                        72114, 133122, 229764, 165405, 266305, 66324, 252039, 342948, 0])
        self.o = np.repeat(np.array(range(1, 10)), 9)
        self.d = np.tile(np.array(range(1, 10)), 9)
        self.dij = np.array([0, 219, 1009, 1514, 974, 1268, 1795, 2420, 3174,
                            219, 0, 831, 1336, 755, 1049, 1576, 2242, 2996,
                            1009, 831, 0, 505, 1019, 662, 933, 1451, 2205,
                            1514, 1336, 505, 0, 1370, 888, 654, 946, 1700,
                            974, 755, 1019, 1370, 0, 482, 1144, 2278, 2862,
                            1268, 1049, 662, 888, 482, 0, 662, 1795, 2380,
                            1795, 1576, 933, 654, 1144, 662, 0, 1287, 1779,
                            2420, 2242, 1451, 946, 2278, 1795, 1287, 0, 754,
                            3147, 2996, 2205, 1700, 2862, 2380, 1779, 754, 0])
        self.dt = pd.DataFrame({'Origin': self.o,
                                'Destination': self.d,
                                'flows': self.f,
                                'Dij': self.dij})
    def test_multiple_parameter(self):
        model = grav.DoublyConstrained(self.dt, 'Origin', 'Destination', 'flows', 'Dij', 'exp')
        ss = {'obs_mean_trip_len': 1250.9555521611339,
              'pred_mean_trip_len': 1250.9555521684863,
              'OD_pairs': 72, 'predicted_flows': 12314322.0,
              'avg_dist_trav': 1251.0, 'num_destinations': 9,
              'observed_flows': 12314322, 'avg_dist': 1414.0,
              'num_origins': 9}
        ps = {'beta': {'LL_zero_val': -4.1172103581711941,
                       'relative_likelihood_stat': 2053596.3814015209,
                       'standard_error': 4.9177433418433932e-07},
                        'all_params': {'zero_vals_LL': -4.1172102183395936,
                        'mle_vals_LL': -4.0338279201692675}}
        fs = {'r_squared': 0.89682406680906979,
              'srmse': 0.24804939821988789}
        es = {'pred_obs_deviance': 0.0314,
              'entropy_ratio': 0.8855,
              'maximum_entropy': 4.2767,
              'max_pred_deviance': 0.2429,
              'variance_obs_entropy': 3.667e-08,
              'predicted_entropy': 4.0338,
              't_stat_entropy': 117.1593,
              'max_obs_deviance': 0.2743,
              'observed_entropy': 4.0024,
              'variance_pred_entropy': 3.516e-08}
        sys_stats = stats.sys_stats(model)
        self.assertAlmostEqual(model.system_stats['obs_mean_trip_len'], ss['obs_mean_trip_len'], 4)
        self.assertAlmostEqual(model.system_stats['pred_mean_trip_len'], ss['pred_mean_trip_len'], 4)
        self.assertAlmostEqual(model.system_stats['OD_pairs'], ss['OD_pairs'])
        self.assertAlmostEqual(model.system_stats['predicted_flows'], ss['predicted_flows'])
        self.assertAlmostEqual(model.system_stats['avg_dist_trav'], ss['avg_dist_trav'])
        self.assertAlmostEqual(model.system_stats['num_destinations'], ss['num_destinations'])
        self.assertAlmostEqual(model.system_stats['observed_flows'], ss['observed_flows'])
        self.assertAlmostEqual(model.system_stats['avg_dist'], ss['avg_dist'], 4)
        self.assertAlmostEqual(model.system_stats['num_origins'], ss['num_origins'])
        param_stats = stats.param_stats(model)
        self.assertAlmostEqual(model.parameter_stats['beta']['LL_zero_val'], ps['beta']['LL_zero_val'], 4)
        self.assertAlmostEqual(model.parameter_stats['beta']['relative_likelihood_stat'],
                                                  ps['beta']['relative_likelihood_stat'], 4)
        self.assertAlmostEqual(model.parameter_stats['beta']['standard_error'], ps['beta']['standard_error'], 4)
        self.assertAlmostEqual(model.parameter_stats['all_params']['zero_vals_LL'], ps['all_params']['zero_vals_LL'], 4)
        self.assertAlmostEqual(model.parameter_stats['all_params']['mle_vals_LL'], ps['all_params']['mle_vals_LL'], 4)
        fit_stats = stats.fit_stats(model)
        self.assertAlmostEqual(model.fit_stats['r_squared'], fs['r_squared'], 4)
        self.assertAlmostEqual(model.fit_stats['srmse'], fs['srmse'], 4)
        ent_stats = stats.ent_stats(model)
        self.assertAlmostEqual(model.entropy_stats['pred_obs_deviance'], es['pred_obs_deviance'], 4)
        self.assertAlmostEqual(model.entropy_stats['entropy_ratio'], es['entropy_ratio'], 4)
        self.assertAlmostEqual(model.entropy_stats['maximum_entropy'], es['maximum_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['max_pred_deviance'], es['max_pred_deviance'], 4)
        self.assertAlmostEqual(model.entropy_stats['variance_obs_entropy'], es['variance_obs_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['predicted_entropy'], es['predicted_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['t_stat_entropy'], es['t_stat_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['max_obs_deviance'], es['max_obs_deviance'], 4)
        self.assertAlmostEqual(model.entropy_stats['observed_entropy'], es['observed_entropy'], 4)
        self.assertAlmostEqual(model.entropy_stats['variance_pred_entropy'], es['variance_pred_entropy'], 4)

if __name__ == '__main__':
    unittest.main()


"""
This is a procedural interface to the matplotlib object-oriented
plotting library.

The following plotting commands are provided; the majority have
Matlab(TM) analogs and similar argument.

_Plotting commands
  acorr     - plot the autocorrelation function
  annotate  - annotate something in the figure
  arrow     - add an arrow to the axes
  axes      - Create a new axes
  axhline   - draw a horizontal line across axes
  axvline   - draw a vertical line across axes
  axhspan   - draw a horizontal bar across axes
  axvspan   - draw a vertical bar across axes
  axis      - Set or return the current axis limits
  bar       - make a bar chart
  barh      - a horizontal bar chart
  broken_barh - a set of horizontal bars with gaps
  box       - set the axes frame on/off state
  boxplot   - make a box and whisker plot
  cla       - clear current axes
  clabel    - label a contour plot
  clf       - clear a figure window
  clim      - adjust the color limits of the current image
  close     - close a figure window
  colorbar  - add a colorbar to the current figure
  cohere    - make a plot of coherence
  contour   - make a contour plot
  contourf  - make a filled contour plot
  csd       - make a plot of cross spectral density
  delaxes   - delete an axes from the current figure
  draw      - Force a redraw of the current figure
  errorbar  - make an errorbar graph
  figlegend - make legend on the figure rather than the axes
  figimage  - make a figure image
  figtext   - add text in figure coords
  figure   - create or change active figure
  fill     - make filled polygons
  findobj  - recursively find all objects matching some criteria
  gca      - return the current axes
  gcf      - return the current figure
  gci      - get the current image, or None
  getp      - get a handle graphics property
  grid     - set whether gridding is on
  hist     - make a histogram
  hold     - set the axes hold state
  ioff     - turn interaction mode off
  ion      - turn interaction mode on
  isinteractive - return True if interaction mode is on
  imread   - load image file into array
  imshow   - plot image data
  ishold   - return the hold state of the current axes
  legend   - make an axes legend
  loglog   - a log log plot
  matshow  - display a matrix in a new figure preserving aspect
  pcolor   - make a pseudocolor plot
  pcolormesh - make a pseudocolor plot using a quadrilateral mesh
  pie      - make a pie chart
  plot     - make a line plot
  plot_date - plot dates
  plotfile  - plot column data from an ASCII tab/space/comma delimited file
  pie      - pie charts
  polar    - make a polar plot on a PolarAxes
  psd      - make a plot of power spectral density
  quiver   - make a direction field (arrows) plot
  rc       - control the default params
  rgrids   - customize the radial grids and labels for polar
  savefig  - save the current figure
  scatter  - make a scatter plot
  setp      - set a handle graphics property
  semilogx - log x axis
  semilogy - log y axis
  show     - show the figures
  specgram - a spectrogram plot
  spy      - plot sparsity pattern using markers or image
  stem     - make a stem plot
  subplot  - make a subplot (numrows, numcols, axesnum)
  subplots_adjust - change the params controlling the subplot positions of current figure
  subplot_tool - launch the subplot configuration tool
  suptitle   - add a figure title
  table    - add a table to the plot
  text     - add some text at location x,y to the current axes
  thetagrids - customize the radial theta grids and labels for polar
  title    - add a title to the current axes
  xcorr   - plot the autocorrelation function of x and y
  xlim     - set/get the xlimits
  ylim     - set/get the ylimits
  xticks   - set/get the xticks
  yticks   - set/get the yticks
  xlabel   - add an xlabel to the current axes
  ylabel   - add a ylabel to the current axes

  autumn - set the default colormap to autumn
  bone   - set the default colormap to bone
  cool   - set the default colormap to cool
  copper - set the default colormap to copper
  flag   - set the default colormap to flag
  gray   - set the default colormap to gray
  hot    - set the default colormap to hot
  hsv    - set the default colormap to hsv
  jet    - set the default colormap to jet
  pink   - set the default colormap to pink
  prism  - set the default colormap to prism
  spring - set the default colormap to spring
  summer - set the default colormap to summer
  winter - set the default colormap to winter
  spectral - set the default colormap to spectral

_Event handling

  connect - register an event handler
  disconnect - remove a connected event handler

_Matrix commands

  cumprod   - the cumulative product along a dimension
  cumsum    - the cumulative sum along a dimension
  detrend   - remove the mean or besdt fit line from an array
  diag      - the k-th diagonal of matrix
  diff      - the n-th differnce of an array
  eig       - the eigenvalues and eigen vectors of v
  eye       - a matrix where the k-th diagonal is ones, else zero
  find      - return the indices where a condition is nonzero
  fliplr    - flip the rows of a matrix up/down
  flipud    - flip the columns of a matrix left/right
  linspace  - a linear spaced vector of N values from min to max inclusive
  logspace  - a log spaced vector of N values from min to max inclusive
  meshgrid  - repeat x and y to make regular matrices
  ones      - an array of ones
  rand      - an array from the uniform distribution [0,1]
  randn     - an array from the normal distribution
  rot90     - rotate matrix k*90 degress counterclockwise
  squeeze   - squeeze an array removing any dimensions of length 1
  tri       - a triangular matrix
  tril      - a lower triangular matrix
  triu      - an upper triangular matrix
  vander    - the Vandermonde matrix of vector x
  svd       - singular value decomposition
  zeros     - a matrix of zeros

_Probability

  levypdf   - The levy probability density function from the char. func.
  normpdf   - The Gaussian probability density function
  rand      - random numbers from the uniform distribution
  randn     - random numbers from the normal distribution

_Statistics

  corrcoef  - correlation coefficient
  cov       - covariance matrix
  amax       - the maximum along dimension m
  mean      - the mean along dimension m
  median    - the median along dimension m
  amin       - the minimum along dimension m
  norm      - the norm of vector x
  prod      - the product along dimension m
  ptp       - the max-min along dimension m
  std       - the standard deviation along dimension m
  asum       - the sum along dimension m

_Time series analysis

  bartlett  - M-point Bartlett window
  blackman  - M-point Blackman window
  cohere    - the coherence using average periodiogram
  csd       - the cross spectral density using average periodiogram
  fft       - the fast Fourier transform of vector x
  hamming   - M-point Hamming window
  hanning   - M-point Hanning window
  hist      - compute the histogram of x
  kaiser    - M length Kaiser window
  psd       - the power spectral density using average periodiogram
  sinc      - the sinc function of array x

_Dates

  date2num  - convert python datetimes to numeric representation
  drange    - create an array of numbers for date plots
  num2date  - convert numeric type (float days since 0001) to datetime

_Other

  angle     - the angle of a complex array
  griddata - interpolate irregularly distributed data to a regular grid
  load     - load ASCII data into array
  polyfit   - fit x, y to an n-th order polynomial
  polyval   - evaluate an n-th order polynomial
  roots     - the roots of the polynomial coefficients in p
  save      - save an array to an ASCII file
  trapz     - trapezoidal integration

__end

"""
import sys, warnings

from cbook import flatten, is_string_like, exception_to_str, popd, \
     silent_list, iterable, dedent

import numpy as np
from numpy import ma

from matplotlib import mpl  # pulls in most modules

from matplotlib.dates import date2num, num2date,\
        datestr2num, strpdate2num, drange,\
        epoch2num, num2epoch, mx2num,\
        DateFormatter, IndexDateFormatter, DateLocator,\
        RRuleLocator, YearLocator, MonthLocator, WeekdayLocator,\
        DayLocator, HourLocator, MinuteLocator, SecondLocator,\
        rrule, MO, TU, WE, TH, FR, SA, SU, YEARLY, MONTHLY,\
        WEEKLY, DAILY, HOURLY, MINUTELY, SECONDLY, relativedelta
import matplotlib.dates

# bring all the  symbols in so folks can import them from
# pylab in one fell swoop

from matplotlib.mlab import  window_hanning, window_none,\
        conv, detrend, detrend_mean, detrend_none, detrend_linear,\
        polyfit, polyval, entropy, normpdf, griddata,\
        levypdf, find, trapz, prepca, rem, norm, orth, rank,\
        sqrtm, prctile, center_matrix, rk4, exp_safe, amap,\
        sum_flat, mean_flat, rms_flat, l1norm, l2norm, norm, frange,\
        diagonal_matrix, base_repr, binary_repr, log2, ispower2,\
        bivariate_normal, load, save

from matplotlib.mlab import stineman_interp, slopes, \
    stineman_interp, inside_poly, poly_below, poly_between, \
    is_closed_polygon, path_length, distances_along_curve, vector_lengths

from numpy import *
from numpy.fft import *
from numpy.random import *
from numpy.linalg import *

from matplotlib.mlab import window_hanning, window_none, conv, detrend, demean, \
     detrend_mean, detrend_none, detrend_linear, entropy, normpdf, levypdf, \
     find, longest_contiguous_ones, longest_ones, prepca, prctile, prctile_rank, \
     center_matrix, rk4, bivariate_normal, get_xyz_where, get_sparse_matrix, dist, \
     dist_point_to_segment, segments_intersect, fftsurr, liaupunov, movavg, \
     save, load, exp_safe, \
     amap, rms_flat, l1norm, l2norm, norm_flat, frange, diagonal_matrix, identity, \
     base_repr, binary_repr, log2, ispower2, fromfunction_kw, rem, norm, orth, rank, sqrtm,\
     mfuncC, approx_real, rec_append_field, rec_drop_fields, rec_join, csv2rec, rec2csv, isvector

from matplotlib.pyplot import *

# provide the recommended module abbrevs in the pylab namespace
import matplotlib.pyplot as plt
import numpy as np


from __future__ import print_function
from show3d_balls import *
import argparse
import os
import random
import numpy as np
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
from datasets import PartDataset
from pointnet import PointGen, PointGenC, PointNetAE
import torch.nn.functional as F
import matplotlib.pyplot as plt

#showpoints(np.random.randn(2500,3), c1 = np.random.uniform(0,1,size = (2500)))

parser = argparse.ArgumentParser()

parser.add_argument('--model', type=str, default = '',  help='model path')

opt = parser.parse_args()
print (opt)

ae = PointNetAE(num_points = 2048)
ae.load_state_dict(torch.load(opt.model))

dataset = PartDataset(root = 'shapenetcore_partanno_segmentation_benchmark_v0', class_choice = ['Chair'], classification = True, npoints = 2048)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=64,
                                          shuffle=True, num_workers=1)

ae.cuda()

i,data = enumerate(dataloader, 0).next()
points, _ = data
points = Variable(points)

bs = points.size()[0]
points = points.transpose(2,1)
points = points.cuda()

gen = ae(points)
point_np = gen.transpose(2,1).cpu().data.numpy()

#showpoints(points.transpose(2,1).cpu().data.numpy())
showpoints(point_np)

#sim_noise = Variable(torch.randn(1000, 100))
#points = gen(sim_noise)
#point_np = points.transpose(2,1).data.numpy()
#print(point_np.shape)

#np.savez('gan.npz', points = point_np)


"""Calibration of predicted probabilities."""

# Author: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#         Balazs Kegl <balazs.kegl@gmail.com>
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#         Mathieu Blondel <mathieu@mblondel.org>
#
# License: BSD 3 clause

from __future__ import division
import inspect
import warnings

from math import log
import numpy as np

from scipy.optimize import fmin_bfgs

from .base import BaseEstimator, ClassifierMixin, RegressorMixin, clone
from .preprocessing import LabelBinarizer
from .utils import check_X_y, check_array, indexable, column_or_1d
from .utils.validation import check_is_fitted
from .isotonic import IsotonicRegression
from .svm import LinearSVC
from .cross_validation import check_cv
from .metrics.classification import _check_binary_probabilistic_predictions

class CalibratedClassifierCV(BaseEstimator, ClassifierMixin):
    """Probability calibration with isotonic regression or sigmoid.

    With this class, the base_estimator is fit on the train set of the
    cross-validation generator and the test set is used for calibration.
    The probabilities for each of the folds are then averaged
    for prediction. In case that cv="prefit" is passed to __init__,
    it is it is assumed that base_estimator has been
    fitted already and all data is used for calibration. Note that
    data for fitting the classifier and for calibrating it must be disjpint.

    Read more in the :ref:`User Guide <calibration>`.

    Parameters
    ----------
    base_estimator : instance BaseEstimator
        The classifier whose output decision function needs to be calibrated
        to offer more accurate predict_proba outputs. If cv=prefit, the
        classifier must have been fit already on data.

    method : 'sigmoid' | 'isotonic'
        The method to use for calibration. Can be 'sigmoid' which
        corresponds to Platt's method or 'isotonic' which is a
        non-parameteric approach. It is not advised to use isotonic calibration
        with too few calibration samples (<<1000) since it tends to overfit.
        Use sigmoids (Platt's calibration) in this case.

    cv : integer or cross-validation generator or "prefit", optional
        If an integer is passed, it is the number of folds (default 3).
        Specific cross-validation objects can be passed, see
        sklearn.cross_validation module for the list of possible objects.
        If "prefit" is passed, it is assumed that base_estimator has been
        fitted already and all data is used for calibration.

    Attributes
    ----------
    classes_ : array, shape (n_classes)
        The class labels.

    calibrated_classifiers_: list (len() equal to cv or 1 if cv == "prefit")
        The list of calibrated classifiers, one for each crossvalidation fold,
        which has been fitted on all but the validation fold and calibrated
        on the validation fold.

    References
    ----------
    .. [1] Obtaining calibrated probability estimates from decision trees
           and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001

    .. [2] Transforming Classifier Scores into Accurate Multiclass
           Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002)

    .. [3] Probabilistic Outputs for Support Vector Machines and Comparisons to
           Regularized Likelihood Methods, J. Platt, (1999)

    .. [4] Predicting Good Probabilities with Supervised Learning,
           A. Niculescu-Mizil & R. Caruana, ICML 2005
    """
    def __init__(self, base_estimator=None, method='sigmoid', cv=3):
        self.base_estimator = base_estimator
        self.method = method
        self.cv = cv

    def fit(self, X, y, sample_weight=None):
        """Fit the calibrated model

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,)
            Target values.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.

        Returns
        -------
        self : object
            Returns an instance of self.
        """
        X, y = check_X_y(X, y, accept_sparse=['csc', 'csr', 'coo'],
                         force_all_finite=False)
        X, y = indexable(X, y)
        lb = LabelBinarizer().fit(y)
        self.classes_ = lb.classes_

        # Check that we each cross-validation fold can have at least one
        # example per class
        n_folds = self.cv if isinstance(self.cv, int) \
            else self.cv.n_folds if hasattr(self.cv, "n_folds") else None
        if n_folds and \
           np.any([np.sum(y == class_) < n_folds for class_ in self.classes_]):
            raise ValueError("Requesting %d-fold cross-validation but provided"
                             " less than %d examples for at least one class."
                             % (n_folds, n_folds))

        self.calibrated_classifiers_ = []
        if self.base_estimator is None:
            # we want all classifiers that don't expose a random_state
            # to be deterministic (and we don't want to expose this one).
            base_estimator = LinearSVC(random_state=0)
        else:
            base_estimator = self.base_estimator

        if self.cv == "prefit":
            calibrated_classifier = _CalibratedClassifier(
                base_estimator, method=self.method)
            if sample_weight is not None:
                calibrated_classifier.fit(X, y, sample_weight)
            else:
                calibrated_classifier.fit(X, y)
            self.calibrated_classifiers_.append(calibrated_classifier)
        else:
            cv = check_cv(self.cv, X, y, classifier=True)
            arg_names = inspect.getargspec(base_estimator.fit)[0]
            estimator_name = type(base_estimator).__name__
            if (sample_weight is not None
                    and "sample_weight" not in arg_names):
                warnings.warn("%s does not support sample_weight. Samples"
                              " weights are only used for the calibration"
                              " itself." % estimator_name)
                base_estimator_sample_weight = None
            else:
                base_estimator_sample_weight = sample_weight
            for train, test in cv:
                this_estimator = clone(base_estimator)
                if base_estimator_sample_weight is not None:
                    this_estimator.fit(
                        X[train], y[train],
                        sample_weight=base_estimator_sample_weight[train])
                else:
                    this_estimator.fit(X[train], y[train])

                calibrated_classifier = _CalibratedClassifier(
                    this_estimator, method=self.method)
                if sample_weight is not None:
                    calibrated_classifier.fit(X[test], y[test],
                                              sample_weight[test])
                else:
                    calibrated_classifier.fit(X[test], y[test])
                self.calibrated_classifiers_.append(calibrated_classifier)

        return self

    def predict_proba(self, X):
        """Posterior probabilities of classification

        This function returns posterior probabilities of classification
        according to each class on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The samples.

        Returns
        -------
        C : array, shape (n_samples, n_classes)
            The predicted probas.
        """
        check_is_fitted(self, ["classes_", "calibrated_classifiers_"])
        X = check_array(X, accept_sparse=['csc', 'csr', 'coo'],
                        force_all_finite=False)
        # Compute the arithmetic mean of the predictions of the calibrated
        # classfiers
        mean_proba = np.zeros((X.shape[0], len(self.classes_)))
        for calibrated_classifier in self.calibrated_classifiers_:
            proba = calibrated_classifier.predict_proba(X)
            mean_proba += proba

        mean_proba /= len(self.calibrated_classifiers_)

        return mean_proba

    def predict(self, X):
        """Predict the target of new samples. Can be different from the
        prediction of the uncalibrated classifier.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The samples.

        Returns
        -------
        C : array, shape (n_samples,)
            The predicted class.
        """
        check_is_fitted(self, ["classes_", "calibrated_classifiers_"])
        return self.classes_[np.argmax(self.predict_proba(X), axis=1)]

class _CalibratedClassifier(object):
    """Probability calibration with isotonic regression or sigmoid.

    It assumes that base_estimator has already been fit, and trains the
    calibration on the input set of the fit function. Note that this class
    should not be used as an estimator directly. Use CalibratedClassifierCV
    with cv="prefit" instead.

    Parameters
    ----------
    base_estimator : instance BaseEstimator
        The classifier whose output decision function needs to be calibrated
        to offer more accurate predict_proba outputs. No default value since
        it has to be an already fitted estimator.

    method : 'sigmoid' | 'isotonic'
        The method to use for calibration. Can be 'sigmoid' which
        corresponds to Platt's method or 'isotonic' which is a
        non-parameteric approach based on isotonic regression.

    References
    ----------
    .. [1] Obtaining calibrated probability estimates from decision trees
           and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001

    .. [2] Transforming Classifier Scores into Accurate Multiclass
           Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002)

    .. [3] Probabilistic Outputs for Support Vector Machines and Comparisons to
           Regularized Likelihood Methods, J. Platt, (1999)

    .. [4] Predicting Good Probabilities with Supervised Learning,
           A. Niculescu-Mizil & R. Caruana, ICML 2005
    """
    def __init__(self, base_estimator, method='sigmoid'):
        self.base_estimator = base_estimator
        self.method = method

    def _preproc(self, X):
        n_classes = len(self.classes_)
        if hasattr(self.base_estimator, "decision_function"):
            df = self.base_estimator.decision_function(X)
            if df.ndim == 1:
                df = df[:, np.newaxis]
        elif hasattr(self.base_estimator, "predict_proba"):
            df = self.base_estimator.predict_proba(X)
            if n_classes == 2:
                df = df[:, 1:]
        else:
            raise RuntimeError('classifier has no decision_function or '
                               'predict_proba method.')

        idx_pos_class = np.arange(df.shape[1])

        return df, idx_pos_class

    def fit(self, X, y, sample_weight=None):
        """Calibrate the fitted model

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        y : array-like, shape (n_samples,)
            Target values.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.

        Returns
        -------
        self : object
            Returns an instance of self.
        """
        lb = LabelBinarizer()
        Y = lb.fit_transform(y)
        self.classes_ = lb.classes_

        df, idx_pos_class = self._preproc(X)
        self.calibrators_ = []

        for k, this_df in zip(idx_pos_class, df.T):
            if self.method == 'isotonic':
                calibrator = IsotonicRegression(out_of_bounds='clip')
            elif self.method == 'sigmoid':
                calibrator = _SigmoidCalibration()
            else:
                raise ValueError('method should be "sigmoid" or '
                                 '"isotonic". Got %s.' % self.method)
            calibrator.fit(this_df, Y[:, k], sample_weight)
            self.calibrators_.append(calibrator)

        return self

    def predict_proba(self, X):
        """Posterior probabilities of classification

        This function returns posterior probabilities of classification
        according to each class on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            The samples.

        Returns
        -------
        C : array, shape (n_samples, n_classes)
            The predicted probas. Can be exact zeros.
        """
        n_classes = len(self.classes_)
        proba = np.zeros((X.shape[0], n_classes))

        df, idx_pos_class = self._preproc(X)

        for k, this_df, calibrator in \
                zip(idx_pos_class, df.T, self.calibrators_):
            if n_classes == 2:
                k += 1
            proba[:, k] = calibrator.predict(this_df)

        # Normalize the probabilities
        if n_classes == 2:
            proba[:, 0] = 1. - proba[:, 1]
        else:
            proba /= np.sum(proba, axis=1)[:, np.newaxis]

        # XXX : for some reason all probas can be 0
        proba[np.isnan(proba)] = 1. / n_classes

        # Deal with cases where the predicted probability minimally exceeds 1.0
        proba[(1.0 < proba) & (proba <= 1.0 + 1e-5)] = 1.0

        return proba

def _sigmoid_calibration(df, y, sample_weight=None):
    """Probability Calibration with sigmoid method (Platt 2000)

    Parameters
    ----------
    df : ndarray, shape (n_samples,)
        The decision function or predict proba for the samples.

    y : ndarray, shape (n_samples,)
        The targets.

    sample_weight : array-like, shape = [n_samples] or None
        Sample weights. If None, then samples are equally weighted.

    Returns
    -------
    a : float
        The slope.

    b : float
        The intercept.

    References
    ----------
    Platt, "Probabilistic Outputs for Support Vector Machines"
    """
    df = column_or_1d(df)
    y = column_or_1d(y)

    F = df  # F follows Platt's notations
    tiny = np.finfo(np.float).tiny  # to avoid division by 0 warning

    # Bayesian priors (see Platt end of section 2.2)
    prior0 = float(np.sum(y <= 0))
    prior1 = y.shape[0] - prior0
    T = np.zeros(y.shape)
    T[y > 0] = (prior1 + 1.) / (prior1 + 2.)
    T[y <= 0] = 1. / (prior0 + 2.)
    T1 = 1. - T

    def objective(AB):
        # From Platt (beginning of Section 2.2)
        E = np.exp(AB[0] * F + AB[1])
        P = 1. / (1. + E)
        l = -(T * np.log(P + tiny) + T1 * np.log(1. - P + tiny))
        if sample_weight is not None:
            return (sample_weight * l).sum()
        else:
            return l.sum()

    def grad(AB):
        # gradient of the objective function
        E = np.exp(AB[0] * F + AB[1])
        P = 1. / (1. + E)
        TEP_minus_T1P = P * (T * E - T1)
        if sample_weight is not None:
            TEP_minus_T1P *= sample_weight
        dA = np.dot(TEP_minus_T1P, F)
        dB = np.sum(TEP_minus_T1P)
        return np.array([dA, dB])

    AB0 = np.array([0., log((prior0 + 1.) / (prior1 + 1.))])
    AB_ = fmin_bfgs(objective, AB0, fprime=grad, disp=False)
    return AB_[0], AB_[1]

class _SigmoidCalibration(BaseEstimator, RegressorMixin):
    """Sigmoid regression model.

    Attributes
    ----------
    a_ : float
        The slope.

    b_ : float
        The intercept.
    """
    def fit(self, X, y, sample_weight=None):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape (n_samples,)
            Training data.

        y : array-like, shape (n_samples,)
            Training target.

        sample_weight : array-like, shape = [n_samples] or None
            Sample weights. If None, then samples are equally weighted.

        Returns
        -------
        self : object
            Returns an instance of self.
        """
        X = column_or_1d(X)
        y = column_or_1d(y)
        X, y = indexable(X, y)

        self.a_, self.b_ = _sigmoid_calibration(X, y, sample_weight)
        return self

    def predict(self, T):
        """Predict new data by linear interpolation.

        Parameters
        ----------
        T : array-like, shape (n_samples,)
            Data to predict from.

        Returns
        -------
        T_ : array, shape (n_samples,)
            The predicted data.
        """
        T = column_or_1d(T)
        return 1. / (1. + np.exp(self.a_ * T + self.b_))

def calibration_curve(y_true, y_prob, normalize=False, n_bins=5):
    """Compute true and predicted probabilities for a calibration curve.

    Read more in the :ref:`User Guide <calibration>`.

    Parameters
    ----------
    y_true : array, shape (n_samples,)
        True targets.

    y_prob : array, shape (n_samples,)
        Probabilities of the positive class.

    normalize : bool, optional, default=False
        Whether y_prob needs to be normalized into the bin [0, 1], i.e. is not
        a proper probability. If True, the smallest value in y_prob is mapped
        onto 0 and the largest one onto 1.

    n_bins : int
        Number of bins. A bigger number requires more data.

    Returns
    -------
    prob_true : array, shape (n_bins,)
        The true probability in each bin (fraction of positives).

    prob_pred : array, shape (n_bins,)
        The mean predicted probability in each bin.

    References
    ----------
    Alexandru Niculescu-Mizil and Rich Caruana (2005) Predicting Good
    Probabilities With Supervised Learning, in Proceedings of the 22nd
    International Conference on Machine Learning (ICML).
    See section 4 (Qualitative Analysis of Predictions).
    """
    y_true = column_or_1d(y_true)
    y_prob = column_or_1d(y_prob)

    if normalize:  # Normalize predicted values into interval [0, 1]
        y_prob = (y_prob - y_prob.min()) / (y_prob.max() - y_prob.min())
    elif y_prob.min() < 0 or y_prob.max() > 1:
        raise ValueError("y_prob has values outside [0, 1] and normalize is "
                         "set to False.")

    y_true = _check_binary_probabilistic_predictions(y_true, y_prob)

    bins = np.linspace(0., 1. + 1e-8, n_bins + 1)
    binids = np.digitize(y_prob, bins) - 1

    bin_sums = np.bincount(binids, weights=y_prob, minlength=len(bins))
    bin_true = np.bincount(binids, weights=y_true, minlength=len(bins))
    bin_total = np.bincount(binids, minlength=len(bins))

    nonzero = bin_total != 0
    prob_true = (bin_true[nonzero] / bin_total[nonzero])
    prob_pred = (bin_sums[nonzero] / bin_total[nonzero])

    return prob_true, prob_pred


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib.patches import *
import pdb

print("""
MatPlotLib Advanced Tutorial
----------------------------
This is a tutorial covering the features and usage of the matplotlib package
in more detail. In truth, no single tutorial can cover all the features that
exist in the matplotlib package since it is extremely expansive. This tutorial
will cover as much material as possible to let you know of the features that
are available to you when plotting data.

Some Notes:
1) A few parts of this program uses pdb to pause the program and allow the
   user to try making things for themselves. Use c to continue with the
   program.

2) This program uses plt for the reference name of pyplot. All pyplot methods
   should be preceded with the qualifier plt such as plt.show().

3) For the best results, run this program with ipython. Regular python may
   dislike plotting during program execution.
""")

pause = input("Press [Enter] to continue...")
print('\n'*100)

print("""
###
## pyplot
###

Within the matplotlib package, the main module you want to be using is the
pyplot module. It is generally imported as

import matplotlib.pyplot as plt

The pyplot module has many useful functions that can be called and used and
we will go over them one by one. For reference, some useful methods are shown
below.

>>> plt.close()              # Closes the current figure. Optional arguments
                               include passing in a figure, figure number,
                               or the string 'all' which closes all figures.
>>> plt.draw()               # Forces the figure to be redrawn. Useful if it
                               was been updated after it was last shown or
                               drawn.
>>> plt.gca()                # Returns the currently active axes object
>>> plt.gcf()                # Returns the currently active figure object
>>> plt.show()               # Shows the latest figure. By default, matplotlib
                               pauses and waits for the window to be closed
                               before continuing. This feature can be turned
                               off with the keyword block = False.
>>> plt.savefig('title.png') # Saves the figure to a file. The file type is
                               automatically determined by the extension.
                               Supported formats include png, pdf, ps, eps,
                               and svg. This has the keywords dpi which
                               specifies the resolution of the output and
                               bbox_inches which, when set to 'tight' reduces
                               any extra white space in the saved file.
>>> plt.subplots_adjust()    # Allows for adjusting parameters of the layout
                               such as the horizontal space (hspace) or width
                               space (wspace) between plots, as well as the
                               left, right, top, and bottom padding.
""")

pause = input("Press [Enter] to continue...")
print('\n'*100)

print("""
###
## Components of a Plot
###

At it's core, matplotlib is nothing more than a graphics package. Every
component of a plot is just a particular "Artist", all drawn on top of
each other to make a nice looking plot.

The beauty of pyplot is the degree of customization that you can have.
You have control over every individual component of this plot and you can
change each of them individually. To do this properly, we will focus on
using the object oriented feature of matplotlib.

Before we talk about how to work with all these features, we need to know
what they are. A window should have just popped up that you can examine.
This window shows all the various components of a figure and the names that
pyplot uses for them. This figure contains the following components

-> Figure    The main part of the plot which everything is shown on. This
             encompasses the entire area of the window, excluding the toolbar.
-> Axes      A single plot, added to the figure. This can have many sets of
             data added to it along with other components such as legends.
             Axes can even sit on top of other axes, but importantly, they
             are still a component of figure, not the axes they may sit inside
             of.
-> Axis      Note the difference here! This is an axIs not an axEs. This
             component is a single axis on the axes and defines how the data
             is plotted. An axes, by default has two axises, the x and y
             (unless you're plotting in 3D in which case it has a z). You can
             add more axises though. Each axis has various components such as
             the spine, tick labels, major ticks, and minor ticks.
-> Spine     Each axis has various components. One of them is the spine. This
             is the actual black line at the border of the plots that the
             tick marks are drawn on. Each default axis has 2 spines. For the
             x axis, it has the top and bottom spine and likewise the y axis
             has the right and left.
-> Legend    Each axes can have a legend added to it. The legend can have lines
             on it, one for each curve labeled.
""")

x = np.arange(0,4*np.pi+np.pi/8,np.pi/8)
y1 = np.sin(x)
y2 = np.cos(x)

fig, (ax1, ax2) = plt.subplots(2, figsize = (10,7))
fig.canvas.set_window_title('Pyplot Figure Components')

plt.subplots_adjust(hspace = 0.4)
plt.suptitle('Figure title', fontsize = 20)

#Create subplot 1
ax1.plot(x, y1, '-dr', label = '$sin(x)$')
ax1.plot(x, np.array([0]*len(x)), '--k')
ax1.set_xlim([0,4*np.pi])
ax1.set_title('Axes 1 Title')
ax1.set_xlabel('Axes 1 x-axis label')
ax1.set_ylabel('Axes 1 y-axis label')
ax1.legend(loc = 'best')

#Create subplot 2
ax2.plot(x, y2, ':og', label = '$cos(x)$')
ax2.plot(x, np.array([0]*len(x)), '-k')
ax2.set_xlim([0,4*np.pi])
ax2.set_title('Axes 2 Title')
ax2.set_xlabel('Axes 2 x-axis label')
ax2.set_ylabel('Axes 2 y-axis label')
ax2.legend(loc = 'best')

#Add artists
ax = fig.add_axes([0,0,1,1])
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
ax.set_zorder(0)
ax.set_axis_bgcolor((0, 0, 0, 0))

ax.add_patch(Rectangle((0.01,0.01),0.98,0.98, fill = False, lw = 2, ec = 'b', transform=ax.transAxes))
ax.annotate('Figure', (0.02,0.02), textcoords = 'axes fraction',
            fontsize = 20, color = 'b', transform=ax.transAxes)

ax.add_patch(Rectangle((0.04,0.5),0.9,0.44, fill = False, lw = 2, ec = 'g', transform=ax.transAxes))
ax.annotate('Axes', (0.05,0.52), textcoords = 'axes fraction',
            fontsize = 20, color = 'g', transform=ax.transAxes)

ax.add_patch(Rectangle((0.11,0.08),0.03,0.38, fill = False, lw = 2, ec = 'r', transform=ax.transAxes))
ax.annotate('Axis', (0.045,0.4), textcoords = 'axes fraction',
            fontsize = 20, color = 'r', transform=ax.transAxes)

ax.add_patch(Rectangle((0.11,0.08),0.8,0.04, fill = False, lw = 2, ec = 'r', transform=ax.transAxes))
ax.annotate('Axis', (0.85,0.04), textcoords = 'axes fraction',
            fontsize = 20, color = 'r')

ax.annotate('Spine', (0.8,0.43), xytext = (0.8,0.35), xycoords = 'axes fraction',
            color = (1,0.5,0), fontsize = 20,
            textcoords = 'axes fraction', horizontalalignment='left',
            arrowprops=dict(arrowstyle = '-|>', fc=(1,0.5,0)))

ax.annotate('', (0.9,0.32), xytext = (0.84,0.34), xycoords = 'axes fraction',
            arrowprops=dict(arrowstyle = '-|>', fc=(1,0.5,0)))

plt.show(block = False)
plt.pause(0.01)

pause = input('Press [Enter] to continue...')
print('\n'*100)

print("""
###
## Objects in matplotlib
###

The above mentioned components of a figure (along with a few others) are
all representable as objects. These objects are stored in a variable which
maintains the state of that object and also has functions the object can
call to change its state. Let's look at how we can use these objects to
create a new figure.
""")

pause = input('Press [Enter] to continue...')
print('\n'*100)

print("""
###
## Creating a New Figure
###

There multiple ways to create a new figure. Probably the simplest is

>>> fig = figure(1, figsize = (5,5), tight_layout = True)

The 1 in this case is an ID for the figure (much like the logical unit
number in IDL). The keywords figsize and tight_layout are optional. The
former sets the physical size of the figure and the second tells the layout
manager to make the plots as close as possible. The state of the figure is
stored in the fig variable which knows entirely about this new figure.
Calling this figure method tells matplotlib that any subsequent plotting
commands should apply to this figure. We can switch to plotting on a new
figure by calling the figure command for another figure (or even switch
back to an old figure). Another method for creating figures is the following

>>> fig = plt.subplots()

This method is much more powerful, but these features will be discussed in
the next section. For reference here are a set of methods and their
functionality that the figure object can call

>>> fig.add_subplot(111)   # Adds a subplot at the specified position
>>> fig.clear()            # Clears the figure's axes
>>> fig.suptitle('Title')  # Adds a title to the figure

Many of the methods listed above as pyplot methods, such as subplots_adjust or
draw, can be applied to a specific figure as well.
""")

pause = input('Press [Enter] to continue...')
print('\n'*100)

print("""
###
## Creating a New Axes
###

Once you have a figure, it's time to add some Axeses to it. As mentioned
before, matplotlib supports using objects. If you've properly created your
figure, it will have been stored into an object. You can now call the method
add_subplot.

>>> ax = fig.add_subplot(1,1,1)

The order of these parameters is add_subplot(rows, columns, plotNo), where
plotNo is the number of the plot, starting at 1 in the upper left and counting
left to right then top to bottom. If all values are less than 10, an equivalent
procedure is to do

>>> ax = fig.add_subplot(111)

Note how this function has created and returned an axes object which we have
stored into the variable ax. There is another method which creates the figure
an axes at the same time

>>> fig, (ax1, ax2) = plt.subplots(nrows = 2, ncols = 1, figsize = (8,8))

The figure is stored into the first variable and the axes are stored into
the second variable with is a tuple of axes.

You can also call the plt.subplot() which acts like add_subplot() but adds
an axes to the currently active figure (determined by the last one referenced).

For more control over your axes positioning, you can specify the exact position
and extent of an axes with the subplot2grid function.

>>> ax = plt.subplot2grid((2,3),(1,0), colspan = 2, rowspan = 1)

This tells figure that there will be a grid of 2 x 3 plots (2 rows, 3 cols) and
this creates a new plot at the position (1,0) (second row, first column) with a
column span of 2 and a row span of 1. If you really want to specify the exact
location, try the add_axes method.

>>> ax = fig.add_axes([0.5, 0.5, 0.3, 0.3])

This tells the figure to put the lower left corner of the axes at the position
(0.5, 0.5) (as fractions of the figure size) and have it span a width and height
of (0.3, 0.3). This is useful to putting plots inside plots. Try this out for
yourself!
""")

pdb.set_trace()
print('\n'*100)

print("""
###
## Plotting to Axes
###

There are many types of plots that can be put on an axes. Below are some simple
examples.

>>> ax.plot()     # Simple scatter/line plot
>>> ax.bar()      # Vertical bar plot
>>> ax.barh()     # Horizonatal bar plot
>>> ax.boxplot()  # Box and wisker plot
>>> ax.contour()  # Contour plot of lines
>>> ax.contourf() # Filled contour plot
>>> ax.errorbar() # Scatter/line plot with errorbars
>>> ax.fill()     # Scatter/line plot which is filled below the curve
>>> ax.hist()     # A histogram of the input data
>>> ax.loglog()   # Scatter/line plot that is logarithmic on both axes
>>> ax.pie()      # Pie chart
>>> ax.polar()    # Polar plot
>>> ax.quiver()   # 2D field of arrows
>>> ax.semilogx() # Scatter/line plot with logarithmic x and linear y.
>>> ax.semilogy() # Equivalent to semilogx, but now y is logarithmic
>>> ax.steamplot()# A streamline of a vector flow
>>> ax.step()     # A step plot

Feel free to try out some of these. You may have to look up the proper
procedures online.
""")

pdb.set_trace()
print('\n'*100)

print("""
###
## Axes Methods
###

Aside from the many plots, there are many useful methods to adjust the
properties of of the axes

>>> ax.add_patch()     # Adds a 'patch' which is an artist like arrows or circles
>>> ax.annotate()      # Adds a string with an arrow to the axes
>>> ax.axhspan()       # Adds a horizontal bar across the plot
>>> ax.axvspan()       # Adds a vertical bar across the plot
>>> ax.arrow()         # Adds an arrow
>>> ax.cla()           # Clears the axes
>>> ax.colorbar()      # Colorbar added to the plot
>>> ax.grid()          # Turns on grid lines, keywords include which (major or
                         minor) and axis (both, x, or y).
>>> ax.legend()        # Legend added to the plot
>>> ax.minorticks_on() # Turns on minor tick marks
>>> ax.set_cmap()      # Sets the color map of the axes
>>> ax.set_title()     # Sets the title of the axes
>>> ax.set_xlabel()    # Sets the x label of the axes
>>> ax.set_xlim()      # Sets the x limits of the axes
>>> ax.set_xscale()    # Sets the scale, either linear, log, or symlog
>>> ax.set_xticklabels()#A list of strings to use for the tick labels
>>> ax.set_xticks()    # Set's values of tick marks with list
## The above x axis specific functions have analagous y axis functions
>>> ax.text()          # Adds a string to the axes
>>> ax.tick_params()   # Changes tick and tick label appearances

Try playing with these various features after creating a figure and axes.
""")

pdb.set_trace()
print('\n'*100)

print("""
###
## Axis and Spines
###

Just as you can work with the specific axes on a figure, you can also work with
specific axis and spines on your axes. These can be extracted and stored in
their own variables, but it is generally easier to refer to them as the
components of the axes object. They are accessed in the following way.

>>> ax.xaxis
>>> ax.yaxis
>>> ax.spine['top']    # Spine is a dict with components, 'top', 'bottom',
                         'left', and 'right'.

These components of the axes have the following useful methods.

>>> ax.xaxis.set_major_formatter()# Set's how the tick marks are formatted
>>> ax.xaxis.set_major_locator()  # Sets location of tick marks (see locators)
## The above major methods have analagous minor methods
>>> ax.xaxis.set_ticklabels()     # Set to empty list to turn off labels
>>> ax.xaxis.set_ticks_position() # Change tick position to only 'top', 'left, etc.
## The above xaxis methods have analagous yaxis methods
>>> ax.spines['top'].set_color()  # Sets the color of the spine
>>> ax.spines['top'].set_position()# Changes position of the spine
>>> ax.spines['top'].set_visible()# Turns off the spine
## The above spine methods have analagous methods for 'bottom', 'left', and
   'right'

Feel free to play with these properties as well.
""")

pdb.set_trace()
print('\n'*100)

print("""
###
## Higher Degrees of Customization
###

We could choose to go even further down the ladder than axis and spines. It is
possible to get the tickmark objects from an axis (via get_major_ticks()) and
change properties on a tickmark by tickmark basis. However, it is no longer
instructive to continue showing methods and ways of doing this as it can always
be looked up. For extreme control over every component of plotting, it is sometimes
useful to use the rcParams variable. This should be imported as

from matplotlib import rcParams

You can then refer to any component of the figure by referencing the dict's
keyword, and setting the value. Common examples include

>>> rcParams['lines.linewidth'] = 2   # Sets linewidths to be 2 by default
>>> rcParams['lines.color'] = 'r'     # Sets line colors to be red by default

There are hundreds of parameters that can be set, all of which can be seen by
going here http://matplotlib.org/users/customizing.html.
""")

pause = input('Press [Enter] to continue...')
print('\n'*100)

print("""
###
## Animations
###

This will only introduce the idea of animations. To actually produce saved
animations in the form of mp4 or some similar format requires installing third
party programs such as ffmpeg. However, matplotlib comes with an animation
package imported as matplotlib.animation. It has tools to allow you to
continually update a plot such that it is animated. There are abilities to
save the animation as well. Below is the code for a very simple animation plot.

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation

fig, ax = plt.subplots()
ax.set_xlim([0,2*np.pi])

x = np.arange(0, 2*np.pi, 0.01)   # x-array
line, = ax.plot(x, np.sin(x))     # The comma after line makes it a tuple

#Init only required for blitting to give a clean slate.
def init():
    line.set_ydata(np.ma.array(x, mask=True))
    return line,

def animate(i):
    line.set_ydata(np.sin(x+i/10.0))  # update the data
    return line,

#blit=True means only redraw the components which have updated. This is
#faster than redrawing everything.
ani = animation.FuncAnimation(fig, animate, init_func=init,
                              interval=25, blit=True)
plt.show()
""")

fig, ax = plt.subplots()
ax.set_xlim([0,2*np.pi])

x = np.arange(0, 2*np.pi, 0.01)   # x-array
line, = ax.plot(x, np.sin(x))     # The comma after line makes it a tuple

#Init only required for blitting to give a clean slate.
def init():
    line.set_ydata(np.ma.array(x, mask=True))
    return line,

def animate(i):
    line.set_ydata(np.sin(x+i/10.0))  # update the data
    return line,

#blit=True means only redraw the components which have updated. This is
#faster than redrawing everything.
ani = animation.FuncAnimation(fig, animate, init_func=init,
                              interval=25, blit=True)
plt.show(block = False)

print('Done...')


#  Copyright 2015-present The Scikit Flow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""
This is an example of using recurrent neural networks over characters
for DBpedia dataset to predict class from description of an entity.

This model is similar to one described in this paper:
   "Character-level Convolutional Networks for Text Classification"
   http://arxiv.org/abs/1509.01626

and is somewhat alternative to the Lua code from here:
   https://github.com/zhangxiangxiao/Crepe
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
from sklearn import metrics
import pandas

import tensorflow as tf
from tensorflow.contrib import learn

### Training data

# Downloads, unpacks and reads DBpedia dataset.
dbpedia = learn.datasets.load_dataset('dbpedia')
X_train, y_train = pandas.DataFrame(dbpedia.train.data)[1], pandas.Series(dbpedia.train.target)
X_test, y_test = pandas.DataFrame(dbpedia.test.data)[1], pandas.Series(dbpedia.test.target)

### Process vocabulary

MAX_DOCUMENT_LENGTH = 100

char_processor = learn.preprocessing.ByteProcessor(MAX_DOCUMENT_LENGTH)
X_train = np.array(list(char_processor.fit_transform(X_train)))
X_test = np.array(list(char_processor.transform(X_test)))

### Models

HIDDEN_SIZE = 20

def char_rnn_model(X, y):
    byte_list = learn.ops.one_hot_matrix(X, 256)
    byte_list = learn.ops.split_squeeze(1, MAX_DOCUMENT_LENGTH, byte_list)
    cell = tf.nn.rnn_cell.GRUCell(HIDDEN_SIZE)
    _, encoding = tf.nn.rnn(cell, byte_list, dtype=tf.float32)
    return learn.models.logistic_regression(encoding, y)

classifier = learn.TensorFlowEstimator(model_fn=char_rnn_model, n_classes=15,
    steps=100, optimizer='Adam', learning_rate=0.01, continue_training=True)

# Continuously train for 1000 steps & predict on test set.
while True:
    classifier.fit(X_train, y_train)
    score = metrics.accuracy_score(y_test, classifier.predict(X_test))
    print("Accuracy: %f" % score)


'''

TestGMMMultTwostepDefault() has lower precision

'''

from statsmodels.compat.python import lmap
import numpy as np
from numpy.testing.decorators import skipif
import pandas
import scipy
from scipy import stats

from statsmodels.regression.linear_model import OLS
from statsmodels.sandbox.regression import gmm

from numpy.testing import assert_allclose, assert_equal
from statsmodels.compat.scipy import NumpyVersion

def get_data():
    import os
    curdir = os.path.split(__file__)[0]
    dt = pandas.read_csv(os.path.join(curdir, 'racd10data_with_transformed.csv'))

    # Transformations compared to original data
    ##dt3['income'] /= 10.
    ##dt3['aget'] = (dt3['age'] - dt3['age'].min()) / 5.
    ##dt3['aget2'] = dt3['aget']**2

    # How do we do this with pandas
    mask = ~((np.asarray(dt['private']) == 1) & (dt['medicaid'] == 1))
    mask = mask & (dt['docvis'] <= 70)
    dt3 = dt[mask]
    dt3['const'] = 1   # add constant
    return dt3

DATA = get_data()

#------------- moment conditions for example

def moment_exponential_add(params, exog, exp=True):

    if not np.isfinite(params).all():
        print("invalid params", params)

    # moment condition without instrument
    if exp:
        predicted = np.exp(np.dot(exog, params))
        #if not np.isfinite(predicted).all():
            #print "invalid predicted", predicted
            #raise RuntimeError('invalid predicted')
        predicted = np.clip(predicted, 0, 1e100)  # try to avoid inf
    else:
        predicted = np.dot(exog, params)

    return predicted

def moment_exponential_mult(params, data, exp=True):
    # multiplicative error model

    endog = data[:,0]
    exog = data[:,1:]

    if not np.isfinite(params).all():
        print("invalid params", params)

    # moment condition without instrument
    if exp:
        predicted = np.exp(np.dot(exog, params))
        predicted = np.clip(predicted, 0, 1e100)  # avoid inf
        resid = endog / predicted - 1
        if not np.isfinite(resid).all():
            print("invalid resid", resid)

    else:
        resid = endog - np.dot(exog, params)

    return resid

#------------------- test classes

# copied from test_gmm.py, with changes
class CheckGMM(object):

    # default tolerance, overwritten by subclasses
    params_tol = [5e-6, 5e-6]
    bse_tol = [5e-7, 5e-7]
    q_tol = [5e-6, 1e-9]
    j_tol = [5e-5, 1e-9]

    def test_basic(self):
        res1, res2 = self.res1, self.res2
        # test both absolute and relative difference
        rtol,  atol = self.params_tol
        assert_allclose(res1.params, res2.params, rtol=rtol, atol=0)
        assert_allclose(res1.params, res2.params, rtol=0, atol=atol)

        rtol,  atol = self.bse_tol
        assert_allclose(res1.bse, res2.bse, rtol=rtol, atol=0)
        assert_allclose(res1.bse, res2.bse, rtol=0, atol=atol)

    def test_other(self):
        res1, res2 = self.res1, self.res2
        rtol,  atol = self.q_tol
        assert_allclose(res1.q, res2.Q, rtol=atol, atol=rtol)
        rtol,  atol = self.j_tol
        assert_allclose(res1.jval, res2.J, rtol=atol, atol=rtol)

        j, jpval, jdf = res1.jtest()
        # j and jval should be the same
        assert_allclose(res1.jval, res2.J, rtol=13, atol=13)
        #pvalue is not saved in Stata results
        pval = stats.chi2.sf(res2.J, res2.J_df)
        #assert_allclose(jpval, pval, rtol=1e-4, atol=1e-6)
        assert_allclose(jpval, pval, rtol=rtol, atol=atol)
        assert_equal(jdf, res2.J_df)

    def test_smoke(self):
        res1 = self.res1
        res1.summary()

class TestGMMAddOnestep(CheckGMM):

    @classmethod
    def setup_class(self):
        XLISTEXOG2 = 'aget aget2 educyr actlim totchr'.split()

        endog_name = 'docvis'
        exog_names = 'private medicaid'.split() + XLISTEXOG2 + ['const']
        instrument_names = 'income ssiratio'.split() + XLISTEXOG2 + ['const']

        endog = DATA[endog_name]
        exog = DATA[exog_names]
        instrument = DATA[instrument_names]

        asarray = lambda x: np.asarray(x, float)
        endog, exog, instrument = lmap(asarray, [endog, exog, instrument])

        self.bse_tol = [5e-6, 5e-7]
        q_tol = [0.04, 0]
        # compare to Stata default options, iterative GMM
        # with const at end
        start = OLS(np.log(endog+1), exog).fit().params
        nobs, k_instr = instrument.shape
        w0inv = np.dot(instrument.T, instrument) / nobs

        mod = gmm.NonlinearIVGMM(endog, exog, instrument, moment_exponential_add)
        res0 = mod.fit(start, maxiter=0, inv_weights=w0inv,
                        optim_method='bfgs', optim_args={'gtol':1e-8, 'disp': 0},
                        wargs={'centered':False})
        self.res1 = res0

        from .results_gmm_poisson import results_addonestep as results
        self.res2 = results

class TestGMMAddTwostep(CheckGMM):

    @classmethod
    def setup_class(self):
        XLISTEXOG2 = 'aget aget2 educyr actlim totchr'.split()

        endog_name = 'docvis'
        exog_names = 'private medicaid'.split() + XLISTEXOG2 + ['const']
        instrument_names = 'income ssiratio'.split() + XLISTEXOG2 + ['const']

        endog = DATA[endog_name]
        exog = DATA[exog_names]
        instrument = DATA[instrument_names]

        asarray = lambda x: np.asarray(x, float)
        endog, exog, instrument = lmap(asarray, [endog, exog, instrument])

        self.bse_tol = [5e-6, 5e-7]
        # compare to Stata default options, iterative GMM
        # with const at end
        start = OLS(np.log(endog+1), exog).fit().params
        nobs, k_instr = instrument.shape
        w0inv = np.dot(instrument.T, instrument) / nobs

        mod = gmm.NonlinearIVGMM(endog, exog, instrument, moment_exponential_add)
        res0 = mod.fit(start, maxiter=2, inv_weights=w0inv,
                        optim_method='bfgs', optim_args={'gtol':1e-8, 'disp': 0},
                        wargs={'centered':False}, has_optimal_weights=False)
        self.res1 = res0

        from .results_gmm_poisson import results_addtwostep as results
        self.res2 = results

class TestGMMMultOnestep(CheckGMM):
    #compares has_optimal_weights=True with Stata's has_optimal_weights=False

    @classmethod
    def setup_class(self):
        # compare to Stata default options, twostep GMM
        XLISTEXOG2 = 'aget aget2 educyr actlim totchr'.split()

        endog_name = 'docvis'
        exog_names = 'private medicaid'.split() + XLISTEXOG2 + ['const']
        instrument_names = 'income medicaid ssiratio'.split() + XLISTEXOG2 + ['const']

        endog = DATA[endog_name]
        exog = DATA[exog_names]
        instrument = DATA[instrument_names]

        asarray = lambda x: np.asarray(x, float)
        endog, exog, instrument = lmap(asarray, [endog, exog, instrument])

        # Need to add all data into exog
        endog_ = np.zeros(len(endog))
        exog_ = np.column_stack((endog, exog))

        self.bse_tol = [5e-6, 5e-7]
        self.q_tol = [0.04, 0]
        self.j_tol = [0.04, 0]
        # compare to Stata default options, iterative GMM
        # with const at end
        start = OLS(endog, exog).fit().params
        nobs, k_instr = instrument.shape
        w0inv = np.dot(instrument.T, instrument) / nobs

        mod = gmm.NonlinearIVGMM(endog_, exog_, instrument, moment_exponential_mult)
        res0 = mod.fit(start, maxiter=0, inv_weights=w0inv,
                        optim_method='bfgs', optim_args={'gtol':1e-8, 'disp': 0},
                        wargs={'centered':False}, has_optimal_weights=False)
        self.res1 = res0

        from .results_gmm_poisson import results_multonestep as results
        self.res2 = results

class TestGMMMultTwostep(CheckGMM):
    #compares has_optimal_weights=True with Stata's has_optimal_weights=False

    @classmethod
    def setup_class(self):
        # compare to Stata default options, twostep GMM
        XLISTEXOG2 = 'aget aget2 educyr actlim totchr'.split()

        endog_name = 'docvis'
        exog_names = 'private medicaid'.split() + XLISTEXOG2 + ['const']
        instrument_names = 'income medicaid ssiratio'.split() + XLISTEXOG2 + ['const']

        endog = DATA[endog_name]
        exog = DATA[exog_names]
        instrument = DATA[instrument_names]

        asarray = lambda x: np.asarray(x, float)
        endog, exog, instrument = lmap(asarray, [endog, exog, instrument])

        # Need to add all data into exog
        endog_ = np.zeros(len(endog))
        exog_ = np.column_stack((endog, exog))

        self.bse_tol = [5e-6, 5e-7]
        # compare to Stata default options, iterative GMM
        # with const at end
        start = OLS(endog, exog).fit().params
        nobs, k_instr = instrument.shape
        w0inv = np.dot(instrument.T, instrument) / nobs

        mod = gmm.NonlinearIVGMM(endog_, exog_, instrument, moment_exponential_mult)
        res0 = mod.fit(start, maxiter=2, inv_weights=w0inv,
                        optim_method='bfgs', optim_args={'gtol':1e-8, 'disp': 0},
                        wargs={'centered':False}, has_optimal_weights=False)
        self.res1 = res0

        from .results_gmm_poisson import results_multtwostep as results
        self.res2 = results

class TestGMMMultTwostepDefault(CheckGMM):
    # compares my defaults with the same options in Stata
    # agreement is not very high, maybe vce(unadjusted) is different after all

    @classmethod
    def setup_class(self):
        # compare to Stata default options, twostep GMM
        XLISTEXOG2 = 'aget aget2 educyr actlim totchr'.split()

        endog_name = 'docvis'
        exog_names = 'private medicaid'.split() + XLISTEXOG2 + ['const']
        instrument_names = 'income medicaid ssiratio'.split() + XLISTEXOG2 + ['const']

        endog = DATA[endog_name]
        exog = DATA[exog_names]
        instrument = DATA[instrument_names]

        asarray = lambda x: np.asarray(x, float)
        endog, exog, instrument = lmap(asarray, [endog, exog, instrument])

        # Need to add all data into exog
        endog_ = np.zeros(len(endog))
        exog_ = np.column_stack((endog, exog))

        self.bse_tol = [0.004, 5e-4]
        self.params_tol = [5e-5, 5e-5]
        # compare to Stata default options, iterative GMM
        # with const at end
        start = OLS(endog, exog).fit().params
        nobs, k_instr = instrument.shape
        w0inv = np.dot(instrument.T, instrument) / nobs

        mod = gmm.NonlinearIVGMM(endog_, exog_, instrument, moment_exponential_mult)
        res0 = mod.fit(start, maxiter=2, inv_weights=w0inv,
                        optim_method='bfgs', optim_args={'gtol':1e-8, 'disp': 0},
                        #wargs={'centered':True}, has_optimal_weights=True
                       )
        self.res1 = res0

        from .results_gmm_poisson import results_multtwostepdefault as results
        self.res2 = results

class TestGMMMultTwostepCenter(CheckGMM):
    #compares my defaults with the same options in Stata

    @classmethod
    def setup_class(self):
        # compare to Stata default options, twostep GMM
        XLISTEXOG2 = 'aget aget2 educyr actlim totchr'.split()

        endog_name = 'docvis'
        exog_names = 'private medicaid'.split() + XLISTEXOG2 + ['const']
        instrument_names = 'income medicaid ssiratio'.split() + XLISTEXOG2 + ['const']

        endog = DATA[endog_name]
        exog = DATA[exog_names]
        instrument = DATA[instrument_names]

        asarray = lambda x: np.asarray(x, float)
        endog, exog, instrument = lmap(asarray, [endog, exog, instrument])

        # Need to add all data into exog
        endog_ = np.zeros(len(endog))
        exog_ = np.column_stack((endog, exog))

        self.bse_tol = [5e-4, 5e-5]
        self.params_tol = [5e-5, 5e-5]
        q_tol = [5e-5, 1e-8]
        # compare to Stata default options, iterative GMM
        # with const at end
        start = OLS(endog, exog).fit().params
        nobs, k_instr = instrument.shape
        w0inv = np.dot(instrument.T, instrument) / nobs

        mod = gmm.NonlinearIVGMM(endog_, exog_, instrument, moment_exponential_mult)
        res0 = mod.fit(start, maxiter=2, inv_weights=w0inv,
                        optim_method='bfgs', optim_args={'gtol':1e-8, 'disp': 0},
                        wargs={'centered':True}, has_optimal_weights=False
                       )
        self.res1 = res0

        from .results_gmm_poisson import results_multtwostepcenter as results
        self.res2 = results

    def test_more(self):

        # from Stata `overid`
        J_df = 1
        J_p = 0.332254330027383
        J = 0.940091427212973

        j, jpval, jdf = self.res1.jtest()

        assert_allclose(jpval, J_p, rtol=5e-5, atol=0)

if __name__ == '__main__':
    tt = TestGMMAddOnestep()
    tt.setup_class()
    tt.test_basic()
    tt.test_other()

    tt = TestGMMAddTwostep()
    tt.setup_class()
    tt.test_basic()
    tt.test_other()

    tt = TestGMMMultOnestep()
    tt.setup_class()
    tt.test_basic()
    #tt.test_other()

    tt = TestGMMMultTwostep()
    tt.setup_class()
    tt.test_basic()
    tt.test_other()

    tt = TestGMMMultTwostepDefault()
    tt.setup_class()
    tt.test_basic()
    tt.test_other()

    tt = TestGMMMultTwostepCenter()
    tt.setup_class()
    tt.test_basic()
    tt.test_other()


import pytest

import pandas

def test_matplotlib_backend_error():
    msg = ('matplotlib is required for plotting when the default backend '
           '"matplotlib" is selected.')
    try:
        import matplotlib  # noqa
    except ImportError:
        with pytest.raises(ImportError, match=msg):
            pandas.set_option('plotting.backend', 'matplotlib')

def test_backend_is_not_module():
    msg = ('"not_an_existing_module" does not seem to be an installed module. '
           'A pandas plotting backend must be a module that can be imported')
    with pytest.raises(ValueError, match=msg):
        pandas.set_option('plotting.backend', 'not_an_existing_module')

def test_backend_is_correct(monkeypatch):
    monkeypatch.setattr('pandas.core.config_init.importlib.import_module',
                        lambda name: None)
    pandas.set_option('plotting.backend', 'correct_backend')
    assert pandas.get_option('plotting.backend') == 'correct_backend'

    # Restore backend for other tests (matplotlib can be not installed)
    try:
        pandas.set_option('plotting.backend', 'matplotlib')
    except ImportError:
        pass


"""
==============================================================
Compute real-time power spectrum density with FieldTrip client
==============================================================

Please refer to `ftclient_rt_average.py` for instructions on
how to get the FieldTrip connector working in MNE-Python.

This example demonstrates how to use it for continuous
computation of power spectra in real-time using the
get_data_as_epoch function.

"""
# Author: Mainak Jas <mainak@neuro.hut.fi>
#
# License: BSD (3-clause)

import numpy as np
import matplotlib.pyplot as plt

import mne
from mne.realtime import FieldTripClient
from mne.time_frequency import compute_epochs_psd

print(__doc__)

# user must provide list of bad channels because
# FieldTrip header object does not provide that
bads = ['MEG 2443', 'EEG 053']

fig, ax = plt.subplots(1)
with FieldTripClient(host='localhost', port=1972,
                     tmax=150, wait_max=10) as rt_client:

    # get measurement info guessed by MNE-Python
    raw_info = rt_client.get_measurement_info()

    # select gradiometers
    picks = mne.pick_types(raw_info, meg='grad', eeg=False, eog=True,
                           stim=False, include=[], exclude=bads)

    n_fft = 256  # the FFT size. Ideally a power of 2
    n_samples = 2048  # time window on which to compute FFT
    for ii in range(20):
        epoch = rt_client.get_data_as_epoch(n_samples=n_samples, picks=picks)
        psd, freqs = compute_epochs_psd(epoch, fmin=2, fmax=200, n_fft=n_fft)

        cmap = 'RdBu_r'
        freq_mask = freqs < 150
        freqs = freqs[freq_mask]
        log_psd = 10 * np.log10(psd[0])

        tmin = epoch.events[0][0] / raw_info['sfreq']
        tmax = (epoch.events[0][0] + n_samples) / raw_info['sfreq']

        if ii == 0:
            im = ax.imshow(log_psd[:, freq_mask].T, aspect='auto',
                           origin='lower', cmap=cmap)

            ax.set_yticks(np.arange(0, len(freqs), 10))
            ax.set_yticklabels(freqs[::10].round(1))
            ax.set_xlabel('Frequency (Hz)')
            ax.set_xticks(np.arange(0, len(picks), 30))
            ax.set_xticklabels(picks[::30])
            ax.set_xlabel('MEG channel index')
            im.set_clim()
        else:
            im.set_data(log_psd[:, freq_mask].T)

        plt.title('continuous power spectrum (t = %0.2f sec to %0.2f sec)'
                  % (tmin, tmax), fontsize=10)

        plt.pause(0.5)
plt.close()