# -*- coding: utf-8 -*-

"""
The :mod:`sklearn.naive_bayes` module implements Naive Bayes algorithms. These
are supervised learning methods based on applying Bayes' theorem with strong
(naive) feature independence assumptions.
"""

# Author: Vincent Michel <vincent.michel@inria.fr>
#         Minor fixes by Fabian Pedregosa
#         Amit Aides <amitibo@tx.technion.ac.il>
#         Yehuda Finkelstein <yehudaf@tx.technion.ac.il>
#         Lars Buitinck <L.J.Buitinck@uva.nl>
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>
#         (parts based on earlier work by Mathieu Blondel)
#
# License: BSD 3 clause

from abc import ABCMeta, abstractmethod

import numpy as np
from scipy.sparse import issparse

from .base import BaseEstimator, ClassifierMixin
from .preprocessing import binarize
from .preprocessing import LabelBinarizer
from .preprocessing import label_binarize
from .utils import check_X_y, check_array
from .utils.extmath import safe_sparse_dot, logsumexp
from .utils.multiclass import _check_partial_fit_first_call
from .utils.fixes import in1d
from .utils.validation import check_is_fitted
from .externals import six

__all__ = ['BernoulliNB', 'GaussianNB', 'MultinomialNB']

class BaseNB(six.with_metaclass(ABCMeta, BaseEstimator, ClassifierMixin)):
    """Abstract base class for naive Bayes estimators"""

    @abstractmethod
    def _joint_log_likelihood(self, X):
        """Compute the unnormalized posterior log probability of X

        I.e. ``log P(c) + log P(x|c)`` for all rows x of X, as an array-like of
        shape [n_classes, n_samples].

        Input is passed to _joint_log_likelihood as-is by predict,
        predict_proba and predict_log_proba.
        """

    def predict(self, X):
        """
        Perform classification on an array of test vectors X.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = [n_samples]
            Predicted target values for X
        """
        jll = self._joint_log_likelihood(X)
        return self.classes_[np.argmax(jll, axis=1)]

    def predict_log_proba(self, X):
        """
        Return log-probability estimates for the test vector X.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array-like, shape = [n_samples, n_classes]
            Returns the log-probability of the samples for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute `classes_`.
        """
        jll = self._joint_log_likelihood(X)
        # normalize by P(x) = P(f_1, ..., f_n)
        log_prob_x = logsumexp(jll, axis=1)
        return jll - np.atleast_2d(log_prob_x).T

    def predict_proba(self, X):
        """
        Return probability estimates for the test vector X.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array-like, shape = [n_samples, n_classes]
            Returns the probability of the samples for each class in
            the model. The columns correspond to the classes in sorted
            order, as they appear in the attribute `classes_`.
        """
        return np.exp(self.predict_log_proba(X))

class GaussianNB(BaseNB):
    """
    Gaussian Naive Bayes (GaussianNB)

    Can perform online updates to model parameters via `partial_fit` method.
    For details on algorithm used to update feature means and variance online,
    see Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque:

        http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf

    Read more in the :ref:`User Guide <gaussian_naive_bayes>`.

    Attributes
    ----------
    class_prior_ : array, shape (n_classes,)
        probability of each class.

    class_count_ : array, shape (n_classes,)
        number of training samples observed in each class.

    theta_ : array, shape (n_classes, n_features)
        mean of each feature per class

    sigma_ : array, shape (n_classes, n_features)
        variance of each feature per class

    Examples
    --------
    >>> import numpy as np
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> Y = np.array([1, 1, 1, 2, 2, 2])
    >>> from sklearn.naive_bayes import GaussianNB
    >>> clf = GaussianNB()
    >>> clf.fit(X, Y)
    GaussianNB()
    >>> print(clf.predict([[-0.8, -1]]))
    [1]
    >>> clf_pf = GaussianNB()
    >>> clf_pf.partial_fit(X, Y, np.unique(Y))
    GaussianNB()
    >>> print(clf_pf.predict([[-0.8, -1]]))
    [1]
    """

    def fit(self, X, y, sample_weight=None):
        """Fit Gaussian Naive Bayes according to X, y

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """
        X, y = check_X_y(X, y)
        return self._partial_fit(X, y, np.unique(y), _refit=True,
                                 sample_weight=sample_weight)

    @staticmethod
    def _update_mean_variance(n_past, mu, var, X, sample_weight=None):
        """Compute online update of Gaussian mean and variance.

        Given starting sample count, mean, and variance, a new set of
        points X, and optionally sample weights, return the updated mean and
        variance. (NB - each dimension (column) in X is treated as independent
        -- you get variance, not covariance).

        Can take scalar mean and variance, or vector mean and variance to
        simultaneously update a number of independent Gaussians.

        See Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque:

        http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf

        Parameters
        ----------
        n_past : int
            Number of samples represented in old mean and variance. If sample
            weights were given, this should contain the sum of sample
            weights represented in old mean and variance.

        mu : array-like, shape (number of Gaussians,)
            Means for Gaussians in original set.

        var : array-like, shape (number of Gaussians,)
            Variances for Gaussians in original set.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        total_mu : array-like, shape (number of Gaussians,)
            Updated mean for each Gaussian over the combined set.

        total_var : array-like, shape (number of Gaussians,)
            Updated variance for each Gaussian over the combined set.
        """
        if X.shape[0] == 0:
            return mu, var

        # Compute (potentially weighted) mean and variance of new datapoints
        if sample_weight is not None:
            n_new = float(sample_weight.sum())
            new_mu = np.average(X, axis=0, weights=sample_weight / n_new)
            new_var = np.average((X - new_mu) ** 2, axis=0,
                                 weights=sample_weight / n_new)
        else:
            n_new = X.shape[0]
            new_var = np.var(X, axis=0)
            new_mu = np.mean(X, axis=0)

        if n_past == 0:
            return new_mu, new_var

        n_total = float(n_past + n_new)

        # Combine mean of old and new data, taking into consideration
        # (weighted) number of observations
        total_mu = (n_new * new_mu + n_past * mu) / n_total

        # Combine variance of old and new data, taking into consideration
        # (weighted) number of observations. This is achieved by combining
        # the sum-of-squared-differences (ssd)
        old_ssd = n_past * var
        new_ssd = n_new * new_var
        total_ssd = (old_ssd + new_ssd +
                     (n_past / float(n_new * n_total)) *
                     (n_new * mu - n_new * new_mu) ** 2)
        total_var = total_ssd / n_total

        return total_mu, total_var

    def partial_fit(self, X, y, classes=None, sample_weight=None):
        """Incremental fit on a batch of samples.

        This method is expected to be called several times consecutively
        on different chunks of a dataset so as to implement out-of-core
        or online learning.

        This is especially useful when the whole dataset is too big to fit in
        memory at once.

        This method has some performance and numerical stability overhead,
        hence it is better to call partial_fit on chunks of data that are
        as large as possible (as long as fitting in the memory budget) to
        hide the overhead.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        classes : array-like, shape (n_classes,)
            List of all the classes that can possibly appear in the y vector.

            Must be provided at the first call to partial_fit, can be omitted
            in subsequent calls.

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """
        return self._partial_fit(X, y, classes, _refit=False,
                                 sample_weight=sample_weight)

    def _partial_fit(self, X, y, classes=None, _refit=False,
                     sample_weight=None):
        """Actual implementation of Gaussian NB fitting.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        classes : array-like, shape (n_classes,)
            List of all the classes that can possibly appear in the y vector.

            Must be provided at the first call to partial_fit, can be omitted
            in subsequent calls.

        _refit: bool
            If true, act as though this were the first time we called
            _partial_fit (ie, throw away any past fitting and start over).

        sample_weight : array-like, shape (n_samples,), optional
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """

        X, y = check_X_y(X, y)
        epsilon = 1e-9

        if _refit:
            self.classes_ = None

        if _check_partial_fit_first_call(self, classes):
            # This is the first call to partial_fit:
            # initialize various cumulative counters
            n_features = X.shape[1]
            n_classes = len(self.classes_)
            self.theta_ = np.zeros((n_classes, n_features))
            self.sigma_ = np.zeros((n_classes, n_features))
            self.class_prior_ = np.zeros(n_classes)
            self.class_count_ = np.zeros(n_classes)
        else:
            if X.shape[1] != self.theta_.shape[1]:
                msg = "Number of features %d does not match previous data %d."
                raise ValueError(msg % (X.shape[1], self.theta_.shape[1]))
            # Put epsilon back in each time
            self.sigma_[:, :] -= epsilon

        classes = self.classes_

        unique_y = np.unique(y)
        unique_y_in_classes = in1d(unique_y, classes)

        if not np.all(unique_y_in_classes):
            raise ValueError("The target label(s) %s in y do not exist in the "
                             "initial classes %s" %
                             (y[~unique_y_in_classes], classes))

        for y_i in unique_y:
            i = classes.searchsorted(y_i)
            X_i = X[y == y_i, :]

            if sample_weight is not None:
                sw_i = sample_weight[y == y_i]
                N_i = sw_i.sum()
            else:
                sw_i = None
                N_i = X_i.shape[0]

            new_theta, new_sigma = self._update_mean_variance(
                self.class_count_[i], self.theta_[i, :], self.sigma_[i, :],
                X_i, sw_i)

            self.theta_[i, :] = new_theta
            self.sigma_[i, :] = new_sigma
            self.class_count_[i] += N_i

        self.sigma_[:, :] += epsilon
        self.class_prior_[:] = self.class_count_ / np.sum(self.class_count_)
        return self

    def _joint_log_likelihood(self, X):
        check_is_fitted(self, "classes_")

        X = check_array(X)
        joint_log_likelihood = []
        for i in range(np.size(self.classes_)):
            jointi = np.log(self.class_prior_[i])
            n_ij = - 0.5 * np.sum(np.log(2. * np.pi * self.sigma_[i, :]))
            n_ij -= 0.5 * np.sum(((X - self.theta_[i, :]) ** 2) /
                                 (self.sigma_[i, :]), 1)
            joint_log_likelihood.append(jointi + n_ij)

        joint_log_likelihood = np.array(joint_log_likelihood).T
        return joint_log_likelihood

class BaseDiscreteNB(BaseNB):
    """Abstract base class for naive Bayes on discrete/categorical data

    Any estimator based on this class should provide:

    __init__
    _joint_log_likelihood(X) as per BaseNB
    """

    def _update_class_log_prior(self, class_prior=None):
        n_classes = len(self.classes_)
        if class_prior is not None:
            if len(class_prior) != n_classes:
                raise ValueError("Number of priors must match number of"
                                 " classes.")
            self.class_log_prior_ = np.log(class_prior)
        elif self.fit_prior:
            # empirical prior, with sample_weight taken into account
            self.class_log_prior_ = (np.log(self.class_count_)
                                     - np.log(self.class_count_.sum()))
        else:
            self.class_log_prior_ = np.zeros(n_classes) - np.log(n_classes)

    def partial_fit(self, X, y, classes=None, sample_weight=None):
        """Incremental fit on a batch of samples.

        This method is expected to be called several times consecutively
        on different chunks of a dataset so as to implement out-of-core
        or online learning.

        This is especially useful when the whole dataset is too big to fit in
        memory at once.

        This method has some performance overhead hence it is better to call
        partial_fit on chunks of data that are as large as possible
        (as long as fitting in the memory budget) to hide the overhead.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target values.

        classes : array-like, shape = [n_classes]
            List of all the classes that can possibly appear in the y vector.

            Must be provided at the first call to partial_fit, can be omitted
            in subsequent calls.

        sample_weight : array-like, shape = [n_samples], optional
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """
        X = check_array(X, accept_sparse='csr', dtype=np.float64)
        _, n_features = X.shape

        if _check_partial_fit_first_call(self, classes):
            # This is the first call to partial_fit:
            # initialize various cumulative counters
            n_effective_classes = len(classes) if len(classes) > 1 else 2
            self.class_count_ = np.zeros(n_effective_classes, dtype=np.float64)
            self.feature_count_ = np.zeros((n_effective_classes, n_features),
                                           dtype=np.float64)
        elif n_features != self.coef_.shape[1]:
            msg = "Number of features %d does not match previous data %d."
            raise ValueError(msg % (n_features, self.coef_.shape[-1]))

        Y = label_binarize(y, classes=self.classes_)
        if Y.shape[1] == 1:
            Y = np.concatenate((1 - Y, Y), axis=1)

        n_samples, n_classes = Y.shape

        if X.shape[0] != Y.shape[0]:
            msg = "X.shape[0]=%d and y.shape[0]=%d are incompatible."
            raise ValueError(msg % (X.shape[0], y.shape[0]))

        # label_binarize() returns arrays with dtype=np.int64.
        # We convert it to np.float64 to support sample_weight consistently
        Y = Y.astype(np.float64)
        if sample_weight is not None:
            sample_weight = np.atleast_2d(sample_weight)
            Y *= check_array(sample_weight).T

        class_prior = self.class_prior

        # Count raw events from data before updating the class log prior
        # and feature log probas
        self._count(X, Y)

        # XXX: OPTIM: we could introduce a public finalization method to
        # be called by the user explicitly just once after several consecutive
        # calls to partial_fit and prior any call to predict[_[log_]proba]
        # to avoid computing the smooth log probas at each call to partial fit
        self._update_feature_log_prob()
        self._update_class_log_prior(class_prior=class_prior)
        return self

    def fit(self, X, y, sample_weight=None):
        """Fit Naive Bayes classifier according to X, y

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vectors, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples]
            Target values.

        sample_weight : array-like, shape = [n_samples], optional
            Weights applied to individual samples (1. for unweighted).

        Returns
        -------
        self : object
            Returns self.
        """
        X, y = check_X_y(X, y, 'csr')
        _, n_features = X.shape

        labelbin = LabelBinarizer()
        Y = labelbin.fit_transform(y)
        self.classes_ = labelbin.classes_
        if Y.shape[1] == 1:
            Y = np.concatenate((1 - Y, Y), axis=1)

        # LabelBinarizer().fit_transform() returns arrays with dtype=np.int64.
        # We convert it to np.float64 to support sample_weight consistently;
        # this means we also don't have to cast X to floating point
        Y = Y.astype(np.float64)
        if sample_weight is not None:
            sample_weight = np.atleast_2d(sample_weight)
            Y *= check_array(sample_weight).T

        class_prior = self.class_prior

        # Count raw events from data before updating the class log prior
        # and feature log probas
        n_effective_classes = Y.shape[1]
        self.class_count_ = np.zeros(n_effective_classes, dtype=np.float64)
        self.feature_count_ = np.zeros((n_effective_classes, n_features),
                                       dtype=np.float64)
        self._count(X, Y)
        self._update_feature_log_prob()
        self._update_class_log_prior(class_prior=class_prior)
        return self

    # XXX The following is a stopgap measure; we need to set the dimensions
    # of class_log_prior_ and feature_log_prob_ correctly.
    def _get_coef(self):
        return (self.feature_log_prob_[1:]
                if len(self.classes_) == 2 else self.feature_log_prob_)

    def _get_intercept(self):
        return (self.class_log_prior_[1:]
                if len(self.classes_) == 2 else self.class_log_prior_)

    coef_ = property(_get_coef)
    intercept_ = property(_get_intercept)

class MultinomialNB(BaseDiscreteNB):
    """
    Naive Bayes classifier for multinomial models

    The multinomial Naive Bayes classifier is suitable for classification with
    discrete features (e.g., word counts for text classification). The
    multinomial distribution normally requires integer feature counts. However,
    in practice, fractional counts such as tf-idf may also work.

    Read more in the :ref:`User Guide <multinomial_naive_bayes>`.

    Parameters
    ----------
    alpha : float, optional (default=1.0)
        Additive (Laplace/Lidstone) smoothing parameter
        (0 for no smoothing).

    fit_prior : boolean
        Whether to learn class prior probabilities or not.
        If false, a uniform prior will be used.

    class_prior : array-like, size (n_classes,)
        Prior probabilities of the classes. If specified the priors are not
        adjusted according to the data.

    Attributes
    ----------
    class_log_prior_ : array, shape (n_classes, )
        Smoothed empirical log probability for each class.

    intercept_ : property
        Mirrors ``class_log_prior_`` for interpreting MultinomialNB
        as a linear model.

    feature_log_prob_ : array, shape (n_classes, n_features)
        Empirical log probability of features
        given a class, ``P(x_i|y)``.

    coef_ : property
        Mirrors ``feature_log_prob_`` for interpreting MultinomialNB
        as a linear model.

    class_count_ : array, shape (n_classes,)
        Number of samples encountered for each class during fitting. This
        value is weighted by the sample weight when provided.

    feature_count_ : array, shape (n_classes, n_features)
        Number of samples encountered for each (class, feature)
        during fitting. This value is weighted by the sample weight when
        provided.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.random.randint(5, size=(6, 100))
    >>> y = np.array([1, 2, 3, 4, 5, 6])
    >>> from sklearn.naive_bayes import MultinomialNB
    >>> clf = MultinomialNB()
    >>> clf.fit(X, y)
    MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
    >>> print(clf.predict(X[2:3]))
    [3]

    Notes
    -----
    For the rationale behind the names `coef_` and `intercept_`, i.e.
    naive Bayes as a linear classifier, see J. Rennie et al. (2003),
    Tackling the poor assumptions of naive Bayes text classifiers, ICML.

    References
    ----------
    C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
    Information Retrieval. Cambridge University Press, pp. 234-265.
    http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html
    """

    def __init__(self, alpha=1.0, fit_prior=True, class_prior=None):
        self.alpha = alpha
        self.fit_prior = fit_prior
        self.class_prior = class_prior

    def _count(self, X, Y):
        """Count and smooth feature occurrences."""
        if np.any((X.data if issparse(X) else X) < 0):
            raise ValueError("Input X must be non-negative")
        self.feature_count_ += safe_sparse_dot(Y.T, X)
        self.class_count_ += Y.sum(axis=0)

    def _update_feature_log_prob(self):
        """Apply smoothing to raw counts and recompute log probabilities"""
        smoothed_fc = self.feature_count_ + self.alpha
        smoothed_cc = smoothed_fc.sum(axis=1)

        self.feature_log_prob_ = (np.log(smoothed_fc)
                                  - np.log(smoothed_cc.reshape(-1, 1)))

    def _joint_log_likelihood(self, X):
        """Calculate the posterior log probability of the samples X"""
        check_is_fitted(self, "classes_")

        X = check_array(X, accept_sparse='csr')
        return (safe_sparse_dot(X, self.feature_log_prob_.T)
                + self.class_log_prior_)

class BernoulliNB(BaseDiscreteNB):
    """Naive Bayes classifier for multivariate Bernoulli models.

    Like MultinomialNB, this classifier is suitable for discrete data. The
    difference is that while MultinomialNB works with occurrence counts,
    BernoulliNB is designed for binary/boolean features.

    Read more in the :ref:`User Guide <bernoulli_naive_bayes>`.

    Parameters
    ----------
    alpha : float, optional (default=1.0)
        Additive (Laplace/Lidstone) smoothing parameter
        (0 for no smoothing).

    binarize : float or None, optional
        Threshold for binarizing (mapping to booleans) of sample features.
        If None, input is presumed to already consist of binary vectors.

    fit_prior : boolean
        Whether to learn class prior probabilities or not.
        If false, a uniform prior will be used.

    class_prior : array-like, size=[n_classes,]
        Prior probabilities of the classes. If specified the priors are not
        adjusted according to the data.

    Attributes
    ----------
    class_log_prior_ : array, shape = [n_classes]
        Log probability of each class (smoothed).

    feature_log_prob_ : array, shape = [n_classes, n_features]
        Empirical log probability of features given a class, P(x_i|y).

    class_count_ : array, shape = [n_classes]
        Number of samples encountered for each class during fitting. This
        value is weighted by the sample weight when provided.

    feature_count_ : array, shape = [n_classes, n_features]
        Number of samples encountered for each (class, feature)
        during fitting. This value is weighted by the sample weight when
        provided.

    Examples
    --------
    >>> import numpy as np
    >>> X = np.random.randint(2, size=(6, 100))
    >>> Y = np.array([1, 2, 3, 4, 4, 5])
    >>> from sklearn.naive_bayes import BernoulliNB
    >>> clf = BernoulliNB()
    >>> clf.fit(X, Y)
    BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)
    >>> print(clf.predict(X[2:3]))
    [3]

    References
    ----------

    C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
    Information Retrieval. Cambridge University Press, pp. 234-265.
    http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html

    A. McCallum and K. Nigam (1998). A comparison of event models for naive
    Bayes text classification. Proc. AAAI/ICML-98 Workshop on Learning for
    Text Categorization, pp. 41-48.

    V. Metsis, I. Androutsopoulos and G. Paliouras (2006). Spam filtering with
    naive Bayes -- Which naive Bayes? 3rd Conf. on Email and Anti-Spam (CEAS).
    """

    def __init__(self, alpha=1.0, binarize=.0, fit_prior=True,
                 class_prior=None):
        self.alpha = alpha
        self.binarize = binarize
        self.fit_prior = fit_prior
        self.class_prior = class_prior

    def _count(self, X, Y):
        """Count and smooth feature occurrences."""
        if self.binarize is not None:
            X = binarize(X, threshold=self.binarize)
        self.feature_count_ += safe_sparse_dot(Y.T, X)
        self.class_count_ += Y.sum(axis=0)

    def _update_feature_log_prob(self):
        """Apply smoothing to raw counts and recompute log probabilities"""
        smoothed_fc = self.feature_count_ + self.alpha
        smoothed_cc = self.class_count_ + self.alpha * 2

        self.feature_log_prob_ = (np.log(smoothed_fc)
                                  - np.log(smoothed_cc.reshape(-1, 1)))

    def _joint_log_likelihood(self, X):
        """Calculate the posterior log probability of the samples X"""
        check_is_fitted(self, "classes_")

        X = check_array(X, accept_sparse='csr')

        if self.binarize is not None:
            X = binarize(X, threshold=self.binarize)

        n_classes, n_features = self.feature_log_prob_.shape
        n_samples, n_features_X = X.shape

        if n_features_X != n_features:
            raise ValueError("Expected input with %d features, got %d instead"
                             % (n_features, n_features_X))

        neg_prob = np.log(1 - np.exp(self.feature_log_prob_))
        # Compute  neg_prob · (1 - X).T  as  ∑neg_prob - X · neg_prob
        jll = safe_sparse_dot(X, (self.feature_log_prob_ - neg_prob).T)
        jll += self.class_log_prior_ + neg_prob.sum(axis=1)

        return jll


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from copy import deepcopy

import numpy as np
import pandas as pd

from xarray import Dataset, DataArray, auto_combine, concat, Variable
from xarray.core.pycompat import iteritems, OrderedDict

from . import TestCase, InaccessibleArray, requires_dask
from .test_dataset import create_test_data

class TestConcatDataset(TestCase):
    def test_concat(self):
        # TODO: simplify and split this test case

        # drop the third dimension to keep things relatively understandable
        data = create_test_data()
        for k in list(data):
            if 'dim3' in data[k].dims:
                del data[k]

        split_data = [data.isel(dim1=slice(3)),
                      data.isel(dim1=slice(3, None))]
        self.assertDatasetIdentical(data, concat(split_data, 'dim1'))

        def rectify_dim_order(dataset):
            # return a new dataset with all variable dimensions transposed into
            # the order in which they are found in `data`
            return Dataset(dict((k, v.transpose(*data[k].dims))
                                for k, v in iteritems(dataset.data_vars)),
                           dataset.coords, attrs=dataset.attrs)

        for dim in ['dim1', 'dim2']:
            datasets = [g for _, g in data.groupby(dim, squeeze=False)]
            self.assertDatasetIdentical(data, concat(datasets, dim))

        dim = 'dim2'
        self.assertDatasetIdentical(
            data, concat(datasets, data[dim]))
        self.assertDatasetIdentical(
            data, concat(datasets, data[dim], coords='minimal'))

        datasets = [g for _, g in data.groupby(dim, squeeze=True)]
        concat_over = [k for k, v in iteritems(data.coords)
                       if dim in v.dims and k != dim]
        actual = concat(datasets, data[dim], coords=concat_over)
        self.assertDatasetIdentical(data, rectify_dim_order(actual))

        actual = concat(datasets, data[dim], coords='different')
        self.assertDatasetIdentical(data, rectify_dim_order(actual))

        # make sure the coords argument behaves as expected
        data.coords['extra'] = ('dim4', np.arange(3))
        for dim in ['dim1', 'dim2']:
            datasets = [g for _, g in data.groupby(dim, squeeze=True)]
            actual = concat(datasets, data[dim], coords='all')
            expected = np.array([data['extra'].values
                                 for _ in range(data.dims[dim])])
            self.assertArrayEqual(actual['extra'].values, expected)

            actual = concat(datasets, data[dim], coords='different')
            self.assertDataArrayEqual(data['extra'], actual['extra'])
            actual = concat(datasets, data[dim], coords='minimal')
            self.assertDataArrayEqual(data['extra'], actual['extra'])

        # verify that the dim argument takes precedence over
        # concatenating dataset variables of the same name
        dim = (2 * data['dim1']).rename('dim1')
        datasets = [g for _, g in data.groupby('dim1', squeeze=False)]
        expected = data.copy()
        expected['dim1'] = dim
        self.assertDatasetIdentical(expected, concat(datasets, dim))

    def test_concat_data_vars(self):
        data = Dataset({'foo': ('x', np.random.randn(10))})
        objs = [data.isel(x=slice(5)), data.isel(x=slice(5, None))]
        for data_vars in ['minimal', 'different', 'all', [], ['foo']]:
            actual = concat(objs, dim='x', data_vars=data_vars)
            self.assertDatasetIdentical(data, actual)

    def test_concat_coords(self):
        data = Dataset({'foo': ('x', np.random.randn(10))})
        expected = data.assign_coords(c=('x', [0] * 5 + [1] * 5))
        objs = [data.isel(x=slice(5)).assign_coords(c=0),
                data.isel(x=slice(5, None)).assign_coords(c=1)]
        for coords in ['different', 'all', ['c']]:
            actual = concat(objs, dim='x', coords=coords)
            self.assertDatasetIdentical(expected, actual)
        for coords in ['minimal', []]:
            with self.assertRaisesRegexp(ValueError, 'not equal across'):
                concat(objs, dim='x', coords=coords)

    def test_concat_constant_index(self):
        # GH425
        ds1 = Dataset({'foo': 1.5}, {'y': 1})
        ds2 = Dataset({'foo': 2.5}, {'y': 1})
        expected = Dataset({'foo': ('y', [1.5, 2.5]), 'y': [1, 1]})
        for mode in ['different', 'all', ['foo']]:
            actual = concat([ds1, ds2], 'y', data_vars=mode)
            self.assertDatasetIdentical(expected, actual)
        with self.assertRaisesRegexp(ValueError, 'not equal across datasets'):
            concat([ds1, ds2], 'y', data_vars='minimal')

    def test_concat_size0(self):
        data = create_test_data()
        split_data = [data.isel(dim1=slice(0, 0)), data]
        actual = concat(split_data, 'dim1')
        self.assertDatasetIdentical(data, actual)

        actual = concat(split_data[::-1], 'dim1')
        self.assertDatasetIdentical(data, actual)

    def test_concat_autoalign(self):
        ds1 = Dataset({'foo': DataArray([1, 2], coords=[('x', [1, 2])])})
        ds2 = Dataset({'foo': DataArray([1, 2], coords=[('x', [1, 3])])})
        actual = concat([ds1, ds2], 'y')
        expected = Dataset({'foo': DataArray([[1, 2, np.nan], [1, np.nan, 2]],
                                             dims=['y', 'x'],
                                             coords={'x': [1, 2, 3]})})
        self.assertDatasetIdentical(expected, actual)

    def test_concat_errors(self):
        data = create_test_data()
        split_data = [data.isel(dim1=slice(3)),
                      data.isel(dim1=slice(3, None))]

        with self.assertRaisesRegexp(ValueError, 'must supply at least one'):
            concat([], 'dim1')

        with self.assertRaisesRegexp(ValueError, 'are not coordinates'):
            concat([data, data], 'new_dim', coords=['not_found'])

        with self.assertRaisesRegexp(ValueError, 'global attributes not'):
            data0, data1 = deepcopy(split_data)
            data1.attrs['foo'] = 'bar'
            concat([data0, data1], 'dim1', compat='identical')
        self.assertDatasetIdentical(
            data, concat([data0, data1], 'dim1', compat='equals'))

        with self.assertRaisesRegexp(ValueError, 'encountered unexpected'):
            data0, data1 = deepcopy(split_data)
            data1['foo'] = ('bar', np.random.randn(10))
            concat([data0, data1], 'dim1')

        with self.assertRaisesRegexp(ValueError, 'compat.* invalid'):
            concat(split_data, 'dim1', compat='foobar')

        with self.assertRaisesRegexp(ValueError, 'unexpected value for'):
            concat([data, data], 'new_dim', coords='foobar')

        with self.assertRaisesRegexp(
                ValueError, 'coordinate in some datasets but not others'):
            concat([Dataset({'x': 0}), Dataset({'x': [1]})], dim='z')

        with self.assertRaisesRegexp(
                ValueError, 'coordinate in some datasets but not others'):
            concat([Dataset({'x': 0}), Dataset({}, {'x': 1})], dim='z')

        with self.assertRaisesRegexp(ValueError, 'no longer a valid'):
            concat([data, data], 'new_dim', mode='different')
        with self.assertRaisesRegexp(ValueError, 'no longer a valid'):
            concat([data, data], 'new_dim', concat_over='different')

    def test_concat_promote_shape(self):
        # mixed dims within variables
        objs = [Dataset({}, {'x': 0}), Dataset({'x': [1]})]
        actual = concat(objs, 'x')
        expected = Dataset({'x': [0, 1]})
        self.assertDatasetIdentical(actual, expected)

        objs = [Dataset({'x': [0]}), Dataset({}, {'x': 1})]
        actual = concat(objs, 'x')
        self.assertDatasetIdentical(actual, expected)

        # mixed dims between variables
        objs = [Dataset({'x': [2], 'y': 3}), Dataset({'x': [4], 'y': 5})]
        actual = concat(objs, 'x')
        expected = Dataset({'x': [2, 4], 'y': ('x', [3, 5])})
        self.assertDatasetIdentical(actual, expected)

        # mixed dims in coord variable
        objs = [Dataset({'x': [0]}, {'y': -1}),
                Dataset({'x': [1]}, {'y': ('x', [-2])})]
        actual = concat(objs, 'x')
        expected = Dataset({'x': [0, 1]}, {'y': ('x', [-1, -2])})
        self.assertDatasetIdentical(actual, expected)

        # scalars with mixed lengths along concat dim -- values should repeat
        objs = [Dataset({'x': [0]}, {'y': -1}),
                Dataset({'x': [1, 2]}, {'y': -2})]
        actual = concat(objs, 'x')
        expected = Dataset({'x': [0, 1, 2]}, {'y': ('x', [-1, -2, -2])})
        self.assertDatasetIdentical(actual, expected)

        # broadcast 1d x 1d -> 2d
        objs = [Dataset({'z': ('x', [-1])}, {'x': [0], 'y': [0]}),
                Dataset({'z': ('y', [1])}, {'x': [1], 'y': [0]})]
        actual = concat(objs, 'x')
        expected = Dataset({'z': (('x', 'y'), [[-1], [1]])},
                           {'x': [0, 1], 'y': [0]})
        self.assertDatasetIdentical(actual, expected)

    def test_concat_do_not_promote(self):
        # GH438
        objs = [Dataset({'y': ('t', [1])}, {'x': 1, 't': [0]}),
                Dataset({'y': ('t', [2])}, {'x': 1, 't': [0]})]
        expected = Dataset({'y': ('t', [1, 2])}, {'x': 1, 't': [0, 0]})
        actual = concat(objs, 't')
        self.assertDatasetIdentical(expected, actual)

        objs = [Dataset({'y': ('t', [1])}, {'x': 1, 't': [0]}),
                Dataset({'y': ('t', [2])}, {'x': 2, 't': [0]})]
        with self.assertRaises(ValueError):
            concat(objs, 't', coords='minimal')

    def test_concat_dim_is_variable(self):
        objs = [Dataset({'x': 0}), Dataset({'x': 1})]
        coord = Variable('y', [3, 4])
        expected = Dataset({'x': ('y', [0, 1]), 'y': [3, 4]})
        actual = concat(objs, coord)
        self.assertDatasetIdentical(actual, expected)

    def test_concat_multiindex(self):
        x = pd.MultiIndex.from_product([[1, 2, 3], ['a', 'b']])
        expected = Dataset({'x': x})
        actual = concat([expected.isel(x=slice(2)),
                         expected.isel(x=slice(2, None))], 'x')
        assert expected.equals(actual)
        assert isinstance(actual.x.to_index(), pd.MultiIndex)

class TestConcatDataArray(TestCase):
    def test_concat(self):
        ds = Dataset({'foo': (['x', 'y'], np.random.random((2, 3))),
                      'bar': (['x', 'y'], np.random.random((2, 3)))},
                     {'x': [0, 1]})
        foo = ds['foo']
        bar = ds['bar']

        # from dataset array:
        expected = DataArray(np.array([foo.values, bar.values]),
                             dims=['w', 'x', 'y'], coords={'x': [0, 1]})
        actual = concat([foo, bar], 'w')
        self.assertDataArrayEqual(expected, actual)
        # from iteration:
        grouped = [g for _, g in foo.groupby('x')]
        stacked = concat(grouped, ds['x'])
        self.assertDataArrayIdentical(foo, stacked)
        # with an index as the 'dim' argument
        stacked = concat(grouped, ds.indexes['x'])
        self.assertDataArrayIdentical(foo, stacked)

        actual = concat([foo[0], foo[1]], pd.Index([0, 1])).reset_coords(drop=True)
        expected = foo[:2].rename({'x': 'concat_dim'})
        self.assertDataArrayIdentical(expected, actual)

        actual = concat([foo[0], foo[1]], [0, 1]).reset_coords(drop=True)
        expected = foo[:2].rename({'x': 'concat_dim'})
        self.assertDataArrayIdentical(expected, actual)

        with self.assertRaisesRegexp(ValueError, 'not identical'):
            concat([foo, bar], dim='w', compat='identical')

        with self.assertRaisesRegexp(ValueError, 'not a valid argument'):
            concat([foo, bar], dim='w', data_vars='minimal')

    @requires_dask
    def test_concat_lazy(self):
        import dask.array as da

        arrays = [DataArray(
            da.from_array(InaccessibleArray(np.zeros((3, 3))), 3),
            dims=['x', 'y']) for _ in range(2)]
        # should not raise
        combined = concat(arrays, dim='z')
        self.assertEqual(combined.shape, (2, 3, 3))
        self.assertEqual(combined.dims, ('z', 'x', 'y'))

class TestAutoCombine(TestCase):

    @requires_dask  # only for toolz
    def test_auto_combine(self):
        objs = [Dataset({'x': [0]}), Dataset({'x': [1]})]
        actual = auto_combine(objs)
        expected = Dataset({'x': [0, 1]})
        self.assertDatasetIdentical(expected, actual)

        actual = auto_combine([actual])
        self.assertDatasetIdentical(expected, actual)

        objs = [Dataset({'x': [0, 1]}), Dataset({'x': [2]})]
        actual = auto_combine(objs)
        expected = Dataset({'x': [0, 1, 2]})
        self.assertDatasetIdentical(expected, actual)

        # ensure auto_combine handles non-sorted variables
        objs = [Dataset(OrderedDict([('x', ('a', [0])), ('y', ('a', [0]))])),
                Dataset(OrderedDict([('y', ('a', [1])), ('x', ('a', [1]))]))]
        actual = auto_combine(objs)
        expected = Dataset({'x': ('a', [0, 1]), 'y': ('a', [0, 1])})
        self.assertDatasetIdentical(expected, actual)

        objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'y': [1], 'x': [1]})]
        with self.assertRaisesRegexp(ValueError, 'too many .* dimensions'):
            auto_combine(objs)

        objs = [Dataset({'x': 0}), Dataset({'x': 1})]
        with self.assertRaisesRegexp(ValueError, 'cannot infer dimension'):
            auto_combine(objs)

        objs = [Dataset({'x': [0], 'y': [0]}), Dataset({'x': [0]})]
        with self.assertRaises(KeyError):
            auto_combine(objs)

    @requires_dask  # only for toolz
    def test_auto_combine_previously_failed(self):
        # In the above scenario, one file is missing, containing the data for
        # one year's data for one variable.
        datasets = [Dataset({'a': ('x', [0]), 'x': [0]}),
                    Dataset({'b': ('x', [0]), 'x': [0]}),
                    Dataset({'a': ('x', [1]), 'x': [1]})]
        expected = Dataset({'a': ('x', [0, 1]), 'b': ('x', [0, np.nan])},
                           {'x': [0, 1]})
        actual = auto_combine(datasets)
        self.assertDatasetIdentical(expected, actual)

        # Your data includes "time" and "station" dimensions, and each year's
        # data has a different set of stations.
        datasets = [Dataset({'a': ('x', [2, 3]), 'x': [1, 2]}),
                    Dataset({'a': ('x', [1, 2]), 'x': [0, 1]})]
        expected = Dataset({'a': (('t', 'x'),
                                  [[np.nan, 2, 3], [1, 2, np.nan]])},
                           {'x': [0, 1, 2]})
        actual = auto_combine(datasets, concat_dim='t')
        self.assertDatasetIdentical(expected, actual)

    @requires_dask  # only for toolz
    def test_auto_combine_still_fails(self):
        # concat can't handle new variables (yet):
        # https://github.com/pydata/xarray/issues/508
        datasets = [Dataset({'x': 0}, {'y': 0}),
                    Dataset({'x': 1}, {'y': 1, 'z': 1})]
        with self.assertRaises(ValueError):
            auto_combine(datasets, 'y')

    @requires_dask  # only for toolz
    def test_auto_combine_no_concat(self):
        objs = [Dataset({'x': 0}), Dataset({'y': 1})]
        actual = auto_combine(objs)
        expected = Dataset({'x': 0, 'y': 1})
        self.assertDatasetIdentical(expected, actual)

        objs = [Dataset({'x': 0, 'y': 1}), Dataset({'y': np.nan, 'z': 2})]
        actual = auto_combine(objs)
        expected = Dataset({'x': 0, 'y': 1, 'z': 2})
        self.assertDatasetIdentical(expected, actual)

        data = Dataset({'x': 0})
        actual = auto_combine([data, data, data], concat_dim=None)
        self.assertDatasetIdentical(data, actual)


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Logit function
=========================================================

Show in the plot is how the logistic regression would, in this
synthetic dataset, classify values as either 0 or 1,
i.e. class one or two, using the logit-curve.

"""
print(__doc__)

# Code source: Gael Varoquaux
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model

# this is our test set, it's just a straight line with some
# Gaussian noise
xmin, xmax = -5, 5
n_samples = 100
np.random.seed(0)
X = np.random.normal(size=n_samples)
y = (X > 0).astype(np.float)
X[X > 0] *= 4
X += .3 * np.random.normal(size=n_samples)

X = X[:, np.newaxis]
# run the classifier
clf = linear_model.LogisticRegression(C=1e5)
clf.fit(X, y)

# and plot the result
plt.figure(1, figsize=(4, 3))
plt.clf()
plt.scatter(X.ravel(), y, color='black', zorder=20)
X_test = np.linspace(-5, 10, 300)

def model(x):
    return 1 / (1 + np.exp(-x))
loss = model(X_test * clf.coef_ + clf.intercept_).ravel()
plt.plot(X_test, loss, color='blue', linewidth=3)

ols = linear_model.LinearRegression()
ols.fit(X, y)
plt.plot(X_test, ols.coef_ * X_test + ols.intercept_, linewidth=1)
plt.axhline(.5, color='.5')

plt.ylabel('y')
plt.xlabel('X')
plt.xticks(())
plt.yticks(())
plt.ylim(-.25, 1.25)
plt.xlim(-4, 10)

plt.show()


#
# FRETBursts - A single-molecule FRET burst analysis toolkit.
#
# Copyright (C) 2014 Antonino Ingargiola <tritemio@gmail.com>
#
"""
Module containing automated unit tests for FRETBursts.

Running the tests requires `py.test`.
"""

from __future__ import division
from builtins import range, zip

from collections import namedtuple
import pytest
import numpy as np

try:
    import matplotlib
except ImportError:
    has_matplotlib = False  # OK to run tests without matplotlib
else:
    has_matplotlib = True
    matplotlib.use('Agg')  # but if matplotlib is installed, use Agg

try:
    import numba
except ImportError:
    has_numba = False
else:
    has_numba = True

import fretbursts.background as bg
import fretbursts.burstlib as bl
import fretbursts.burstlib_ext as bext
from fretbursts import loader
from fretbursts import select_bursts
from fretbursts.ph_sel import Ph_sel
from fretbursts.phtools import phrates
if has_matplotlib:
    import fretbursts.burst_plot as bplt

# data subdir in the notebook folder
DATASETS_DIR = u'notebooks/data/'

def _alex_process(d):
    loader.alex_apply_period(d)
    d.calc_bg(bg.exp_fit, time_s=30, tail_min_us=300)
    d.burst_search(L=10, m=10, F=7)

def load_dataset_1ch(process=True):
    fn = "0023uLRpitc_NTP_20dT_0.5GndCl.hdf5"
    fname = DATASETS_DIR + fn
    d = loader.photon_hdf5(fname)
    if process:
        _alex_process(d)
    return d

def load_dataset_8ch():
    fn = "12d_New_30p_320mW_steer_3.hdf5"
    fname = DATASETS_DIR + fn
    d = loader.photon_hdf5(fname)
    d.calc_bg(bg.exp_fit, time_s=30, tail_min_us=300)
    d.burst_search(L=10, m=10, F=7)
    return d

def load_fake_pax():
    fn = "0023uLRpitc_NTP_20dT_0.5GndCl.hdf5"
    fname = DATASETS_DIR + fn
    d = loader.photon_hdf5(fname)
    d.add(ALEX=False, meas_type='PAX')
    loader.alex_apply_period(d)
    d.calc_bg(bg.exp_fit, time_s=30, tail_min_us='auto')
    d.burst_search(L=10, m=10, F=6)
    return d

@pytest.fixture(scope="module", params=[
                                    load_dataset_1ch,
                                    load_dataset_8ch,
                                    ])
def data(request):
    load_func = request.param
    d = load_func()
    return d

@pytest.fixture(scope="module")
def data_8ch(request):
    d = load_dataset_8ch()
    return d

@pytest.fixture(scope="module")
def data_1ch(request):
    d = load_dataset_1ch()
    return d

##
# List comparison functions
#

def list_equal(list1, list2):
    """Test numerical equality of all the elements in the two lists.
    """
    return np.all([val1 == val2 for val1, val2 in zip(list1, list2)])

def list_array_equal(list1, list2):
    """Test numerical equality between two lists of arrays.
    """
    return np.all([np.all(arr1 == arr2) for arr1, arr2 in zip(list1, list2)])

def list_array_allclose(list1, list2):
    """Test float closeness (np.allclose) between two lists of arrays.
    """
    return np.all([np.allclose(arr1, arr2) for arr1, arr2 in zip(list1, list2)])

##
# Test functions
#

def test_bg_compatlayer_for_obsolete_attrs():
    d = load_dataset_1ch(process=False)
    attrs = ('bg_dd', 'bg_ad', 'bg_da', 'bg_aa',
             'rate_m', 'rate_dd', 'rate_ad', 'rate_da', 'rate_aa')
    for attr in attrs:
        with pytest.raises(RuntimeError):
            getattr(d, attr)
    _alex_process(d)
    for attr in attrs:
        assert isinstance(getattr(d, attr), list)

def test_ph_times_compact(data_1ch):
    """Test calculation of ph_times_compact."""
    def isinteger(x):
        return np.equal(np.mod(x, 1), 0)
    ich = 0
    d = data_1ch

    ph_d = d.get_ph_times(ph_sel=Ph_sel(Dex='DAem'))
    ph_a = d.get_ph_times(ph_sel=Ph_sel(Aex='DAem'))
    ph_dc = d.get_ph_times(ph_sel=Ph_sel(Dex='DAem'), compact=True)
    ph_ac = d.get_ph_times(ph_sel=Ph_sel(Aex='DAem'), compact=True)
    # Test that the difference of ph and ph_compact is multiple of
    # the complementary excitation period duration
    Dex_void = bl._excitation_width(d._D_ON_multich[ich], d.alex_period)
    Aex_void = bl._excitation_width(d._A_ON_multich[ich], d.alex_period)
    assert isinteger((ph_d - ph_dc) / Dex_void).all()
    assert isinteger((ph_a - ph_ac) / Aex_void).all()
    # Test that alternation histogram does not have "gaps" for ph_compact
    bins = np.linspace(0, d.alex_period, num=101)
    hist_dc, _ = np.histogram(ph_dc % d.alex_period, bins=bins)
    hist_ac, _ = np.histogram(ph_ac % d.alex_period, bins=bins)
    assert (hist_dc > 0).all()
    assert (hist_ac > 0).all()

def test_time_min_max():
    """Test time_min and time_max for ALEX data."""
    d = load_dataset_1ch(process=False)
    ich = 0
    assert d.time_max == d.ph_times_t[ich].max() * d.clk_p
    assert d.time_min == d.ph_times_t[ich].min() * d.clk_p
    del d._time_max, d._time_min
    _alex_process(d)
    assert d.time_max == d.ph_times_m[ich][-1] * d.clk_p
    assert d.time_min == d.ph_times_m[ich][0] * d.clk_p
    d.delete('ph_times_m')
    del d._time_max, d._time_min
    assert d.time_max == d.mburst[0].stop[-1] * d.clk_p
    assert d.time_min == d.mburst[0].start[0] * d.clk_p

def test_time_min_max_multispot(data_8ch):
    """Test time_min and time_max for multi-spot data."""
    d = data_8ch
    assert d.time_max == max(t[-1] for t in d.ph_times_m) * d.clk_p
    assert d.time_min == min(t[0] for t in d.ph_times_m) * d.clk_p

def test_aex_dex_ratio(data_1ch):
    """Test methods computing relative D and A alternation periods durations.
    """
    d = data_1ch
    Dx, Ax = d.D_ON, d.A_ON
    a1 = d._aex_fraction()
    a2 = (Ax[1] - Ax[0]) / (Ax[1] - Ax[0] + Dx[1] - Dx[0])
    assert a1 == a2
    r1 = d._aex_dex_ratio()
    r2 = (Ax[1] - Ax[0]) / (Dx[1] - Dx[0])
    assert r1 == r2
    assert (a1 / (1 - a1)) == r1

def test_burst_size_pax():
    d = load_fake_pax()
    aex_dex_ratio, alpha_d = d._aex_dex_ratio(), 1 - d._aex_fraction()
    nd, na = d.nd[0], d.na[0]
    nda = d.nda[0]
    naa = d.naa[0] - d.nar[0] * aex_dex_ratio

    # Test burst size during Dex
    b1 = d.burst_sizes_pax_ich(add_aex=False)
    b2 = d.burst_sizes_ich(add_naa=False)
    b3 = nd + na
    assert (b1 == b2).all()
    assert (b1 == b3).all()

    # Test naa
    naa2 = d.get_naa_corrected()
    naa3 = d._get_naa_ich()
    assert (naa == naa2).all()
    assert (naa == naa3).all()

    # Test add_naa
    b1 = d.burst_sizes_ich(add_naa=True)
    b2 = nd + na + naa
    assert (b1 == b2).all()

    # Test add_aex with no duty-cycle correction
    b1 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=False)
    b2 = nd + na + nda + d.naa[0]
    b3 = nd + na + nda + naa + na * aex_dex_ratio
    assert np.allclose(b1, b2)
    assert np.allclose(b1, b3)

    # Test add_aex with duty-cycle correction
    b1 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True)
    b2 = nd + na + nda + na * aex_dex_ratio + naa / alpha_d
    assert np.allclose(b1, b2)

    # Test add_aex with duty-cycle correction, donor_ref
    b1 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True, donor_ref=True)
    b2 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True, donor_ref=False)
    assert np.allclose(b1, b2)

    # Test add_aex with duty-cycle correction, gamma, beta
    gamma = 0.7
    beta = 0.85
    b1 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True,
                               gamma=gamma, beta=beta, donor_ref=True)
    b2 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True,
                               gamma=gamma, beta=beta, donor_ref=False)
    assert np.allclose(b1 * gamma, b2)
    b1 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True,
                               gamma=gamma, beta=beta, donor_ref=False)

    b2 = (gamma * (nd + nda) + na * (1 + aex_dex_ratio) +
          naa / (alpha_d * beta))
    assert np.allclose(b1, b2)

    d.leakage = 0.1
    nd, na = d.nd[0], d.na[0]
    nda = d.nda[0]
    naa = d.naa[0] - d.nar[0] * aex_dex_ratio

    # Test add_aex with duty-cycle correction, gamma, beta
    gamma = 0.7
    beta = 0.85
    b1 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True,
                               gamma=gamma, beta=beta, donor_ref=True)
    b2 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True,
                               gamma=gamma, beta=beta, donor_ref=False)
    assert np.allclose(b1 * gamma, b2)
    b1 = d.burst_sizes_pax_ich(add_aex=True, aex_corr=True,
                               gamma=gamma, beta=beta, donor_ref=False)

    b2 = (gamma * (nd + nda) + na * (1 + aex_dex_ratio) +
          naa / (alpha_d * beta))
    assert np.allclose(b1, b2)

def test_bg_calc(data):
    """Smoke test bg_calc() and test deletion of bg fields.
    """
    data.calc_bg(bg.exp_fit, time_s=30, tail_min_us=300)
    assert 'bg_auto_th_us0' not in data
    assert 'bg_auto_F_bg' not in data
    assert 'bg_th_us_user' in data

    data.calc_bg(bg.exp_fit, time_s=30, tail_min_us='auto', F_bg=1.7)
    assert 'bg_auto_th_us0' in data
    assert 'bg_auto_F_bg' in data
    assert 'bg_th_us_user' not in data

    data.calc_bg(bg.exp_fit, time_s=30, tail_min_us='auto', F_bg=1.7,
                 fit_allph=False)
    streams = [s for s in data.ph_streams if s != Ph_sel('all')]
    bg_t = [np.sum(data.bg[s][ich] for s in streams) for ich in range(data.nch)]
    assert list_array_equal(data.bg[Ph_sel('all')], bg_t)

def test_ph_streams(data):
    sel = [Ph_sel('all'), Ph_sel(Dex='Dem'), Ph_sel(Dex='Aem')]
    if data.alternated:
        sel.extend([Ph_sel(Aex='Aem'), Ph_sel(Aex='Dem')])
    for s in sel:
        assert s in data.ph_streams

def test_bg_from(data):
    """Test the method .bg_from() for all the ph_sel combinations.
    """
    d = data
    for sel in d.ph_streams:
        bg = d.bg_from(ph_sel=sel)
        assert list_array_equal(bg, d.bg[sel])

    if not (data.alternated):
        assert list_array_equal(d.bg_from(Ph_sel('all')),
                                d.bg_from(Ph_sel(Dex='DAem')))
        return

    bg_dd = d.bg_from(ph_sel=Ph_sel(Dex='Dem'))
    bg_ad = d.bg_from(ph_sel=Ph_sel(Dex='Aem'))

    bg = d.bg_from(ph_sel=Ph_sel(Dex='DAem'))
    assert list_array_equal(bg, [b1 + b2 for b1, b2 in zip(bg_dd, bg_ad)])

    bg_aa = d.bg_from(ph_sel=Ph_sel(Aex='Aem'))
    bg_da = d.bg_from(ph_sel=Ph_sel(Aex='Dem'))

    bg = d.bg_from(ph_sel=Ph_sel(Aex='DAem'))
    assert list_array_equal(bg, [b1 + b2 for b1, b2 in zip(bg_aa, bg_da)])

    bg = d.bg_from(ph_sel=Ph_sel(Dex='Dem', Aex='Dem'))
    assert list_array_equal(bg, [b1 + b2 for b1, b2 in zip(bg_dd, bg_da)])

    bg = d.bg_from(ph_sel=Ph_sel(Dex='Aem', Aex='Aem'))
    assert list_array_equal(bg, [b1 + b2 for b1, b2 in zip(bg_ad, bg_aa)])

    bg = d.bg_from(ph_sel=Ph_sel(Dex='DAem'))
    assert list_array_equal(bg, [b1 + b2 for b1, b2 in zip(bg_dd, bg_ad)])

    bg = d.bg_from(ph_sel=Ph_sel(Dex='DAem', Aex='Aem'))
    bg2 = [b1 + b2 + b3 for b1, b2, b3 in zip(bg_dd, bg_ad, bg_aa)]
    assert list_array_equal(bg, bg2)

def test_iter_ph_times(data):
    """Test method .iter_ph_times() for all the ph_sel combinations.
    """
    # TODO add all the ph_sel combinations like in test_bg_from()
    d = data

    assert list_array_equal(d.ph_times_m, d.iter_ph_times())

    for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Dex='Dem'))):
        if d.alternated:
            assert (ph == d.ph_times_m[ich][d.D_em[ich] * d.D_ex[ich]]).all()
        else:
            assert (ph == d.ph_times_m[ich][~d.A_em[ich]]).all()

    for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Dex='Aem'))):
        if d.alternated:
            assert (ph == d.ph_times_m[ich][d.A_em[ich] * d.D_ex[ich]]).all()
        else:
            assert (ph == d.ph_times_m[ich][d.A_em[ich]]).all()

    if d.alternated:
        for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Aex='Dem'))):
            assert (ph == d.ph_times_m[ich][d.D_em[ich] * d.A_ex[ich]]).all()
        for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Aex='Aem'))):
            assert (ph == d.ph_times_m[ich][d.A_em[ich] * d.A_ex[ich]]).all()

        for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Dex='DAem'))):
            assert (ph == d.ph_times_m[ich][d.D_ex[ich]]).all()
        for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Aex='DAem'))):
            assert (ph == d.ph_times_m[ich][d.A_ex[ich]]).all()

        for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Dex='Dem', Aex='Dem'))):
            assert (ph == d.ph_times_m[ich][d.D_em[ich]]).all()
        for ich, ph in enumerate(d.iter_ph_times(Ph_sel(Dex='Aem', Aex='Aem'))):
            assert (ph == d.ph_times_m[ich][d.A_em[ich]]).all()

        for ich, ph in enumerate(d.iter_ph_times(
                Ph_sel(Dex='DAem', Aex='Aem'))):
            mask = d.D_ex[ich] + d.A_em[ich] * d.A_ex[ich]
            assert (ph == d.ph_times_m[ich][mask]).all()
    else:
        assert list_array_equal(d.iter_ph_times(),
                                d.iter_ph_times(Ph_sel(Dex='DAem')))

def test_get_ph_times_period(data):
    for ich in range(data.nch):
        data.get_ph_times_period(0, ich=ich)
        data.get_ph_times_period(0, ich=ich, ph_sel=Ph_sel(Dex='Dem'))

def test_iter_ph_times_period(data):
    d = data
    for ich in range(data.nch):
        for period, ph_period in enumerate(d.iter_ph_times_period(ich=ich)):
            istart, iend = d.Lim[ich][period]
            assert (ph_period == d.ph_times_m[ich][istart : iend + 1]).all()

        ph_sel = Ph_sel(Dex='Dem')
        mask = d.get_ph_mask(ich=ich, ph_sel=ph_sel)
        for period, ph_period in enumerate(
                d.iter_ph_times_period(ich=ich, ph_sel=ph_sel)):
            istart, iend = d.Lim[ich][period]
            ph_period_test = d.ph_times_m[ich][istart : iend + 1]
            ph_period_test = ph_period_test[mask[istart : iend + 1]]
            assert (ph_period == ph_period_test).all()

def test_burst_search_py_cy(data):
    """Test python and cython burst search with background-dependent threshold.
    """
    data.burst_search(pure_python=True)
    mburst1 = [b.copy() for b in data.mburst]
    num_bursts1 = data.num_bursts
    data.burst_search(pure_python=False)
    assert np.all(num_bursts1 == data.num_bursts)
    assert mburst1 == data.mburst
    data.burst_search(L=30, pure_python=True)
    mburst1 = [b.copy() for b in data.mburst]
    num_bursts1 = data.num_bursts
    data.burst_search(L=30, pure_python=False)
    assert np.all(num_bursts1 == data.num_bursts)
    assert mburst1 == data.mburst

def test_burst_search_constant_rates(data):
    """Test python and cython burst search with constant threshold."""
    data.burst_search(min_rate_cps=50e3, pure_python=True)
    assert (data.num_bursts > 0).all()
    mburst1 = [b.copy() for b in data.mburst]
    num_bursts1 = data.num_bursts
    data.burst_search(min_rate_cps=50e3, pure_python=False)
    assert (data.num_bursts > 0).all()
    assert np.all(num_bursts1 == data.num_bursts)
    assert mburst1 == data.mburst

def test_burst_search_L(data):
    """Test burst search with different L arguments."""
    data.burst_search(L=10)
    for bursts in data.mburst:
        assert (bursts.counts >= 10).all()
    num_bursts1 = data.num_bursts
    data.burst_search(L=30)
    for bursts in data.mburst:
        assert (bursts.counts >= 30).all()
    assert np.all(num_bursts1 > data.num_bursts)

def test_burst_search_with_no_bursts(data):
    """Smoke test burst search when some periods have no bursts."""
    # F=600 results in periods with no bursts for the us-ALEX measurement
    # and in no bursts at all for the multi-spot measurements
    data.burst_search(m=10, F=600)
    data.fuse_bursts(ms=1)

if has_matplotlib:
    def test_stale_fitter_after_burst_search(data):
        """Test that E/S_fitter attributes are deleted on burst search."""
        data.burst_search(L=10, m=10, F=7, ph_sel=Ph_sel(Dex='Dem'))
        bplt.dplot(data, bplt.hist_fret)  # create E_fitter attribute
        if data.alternated:
            bplt.dplot(data, bplt.hist_S)  # create S_fitter attribute

        data.burst_search(L=10, m=10, F=7, ph_sel=Ph_sel(Dex='Aem'))
        assert not hasattr(data, 'E_fitter')
        if data.alternated:
            assert not hasattr(data, 'S_fitter')

        bplt.dplot(data, bplt.hist_fret)  # create E_fitter attribute
        if data.alternated:
            bplt.dplot(data, bplt.hist_S)  # create S_fitter attribute

        data.calc_fret()
        assert not hasattr(data, 'E_fitter')
        if data.alternated:
            assert not hasattr(data, 'S_fitter')

def test_burst_search(data):
    """Smoke test and bg_bs check."""
    streams = [Ph_sel(Dex='Dem'), Ph_sel(Dex='Aem')]
    if data.alternated:
        streams.extend([Ph_sel(Dex='Aem', Aex='Aem'), Ph_sel(Dex='DAem')])
    for sel in streams:
        data.burst_search(L=10, m=10, F=7, ph_sel=sel)
        assert list_equal(data.bg_bs, data.bg_from(sel))

    if data.alternated:
        data.burst_search(m=10, F=7, ph_sel=Ph_sel(Dex='DAem'), compact=True)
    data.burst_search(L=10, m=10, F=7)

def test_burst_search_and_gate(data_1ch):
    """Test consistency of burst search and gate."""
    d = data_1ch
    assert d.alternated

    # Smoke tests
    bext.burst_search_and_gate(d, F=(6, 8))
    bext.burst_search_and_gate(d, m=(12, 8))
    bext.burst_search_and_gate(d, min_rate_cps=(60e3, 40e3))
    if d.nch > 1:
        mr1 = 35e3 + np.arange(d.nch) * 1e3
        mr2 = 30e3 + np.arange(d.nch) * 1e3
        bext.burst_search_and_gate(d, min_rate_cps=(mr1, mr2))

    # Consistency test
    d_dex = d.copy()
    d_dex.burst_search(ph_sel=Ph_sel(Dex='DAem'))
    d_aex = d.copy()
    d_aex.burst_search(ph_sel=Ph_sel(Aex='Aem'))
    d_and = bext.burst_search_and_gate(d)
    for bursts_dex, bursts_aex, bursts_and, ph in zip(
            d_dex.mburst, d_aex.mburst, d_and.mburst, d.iter_ph_times()):
        ph_b_mask_dex = bl.ph_in_bursts_mask(ph.size, bursts_dex)
        ph_b_mask_aex = bl.ph_in_bursts_mask(ph.size, bursts_aex)
        ph_b_mask_and = bl.ph_in_bursts_mask(ph.size, bursts_and)
        assert (ph_b_mask_and == ph_b_mask_dex * ph_b_mask_aex).all()

def test_mch_count_ph_num_py_c(data):
    na_py = bl.bslib.mch_count_ph_in_bursts_py(data.mburst, data.A_em)
    na_c = bl.bslib.mch_count_ph_in_bursts_c(data.mburst, data.A_em)
    assert list_array_equal(na_py, na_c)
    assert na_py[0].dtype == np.float64

def test_burst_sizes(data):
    """Test for .burst_sizes_ich() and burst_sizes()"""
    # Smoke test
    plain_sizes = data.burst_sizes()
    assert len(plain_sizes) == data.nch
    # Test gamma and donor_ref arguments
    bs1 = data.burst_sizes_ich(gamma=0.5, donor_ref=True)
    bs2 = data.burst_sizes_ich(gamma=0.5, donor_ref=False)
    assert np.allclose(bs1, bs2 / 0.5)
    # Test add_naa
    if data.alternated:
        bs_no_naa = data.burst_sizes_ich(add_naa=False)
        bs_naa = data.burst_sizes_ich(add_naa=True)
        assert np.allclose(bs_no_naa + data.naa[0], bs_naa)

        # Test beta and donor_ref arguments with gamma=1
        naa1 = data.get_naa_corrected(beta=0.8, donor_ref=True)
        naa2 = data.get_naa_corrected(beta=0.8, donor_ref=False)
        assert np.allclose(naa1, naa2)

        # Test beta and donor_ref arguments with gamma=0.5
        naa1 = data.get_naa_corrected(gamma=0.5, beta=0.8, donor_ref=True)
        naa2 = data.get_naa_corrected(gamma=0.5, beta=0.8, donor_ref=False)
        assert np.allclose(naa1 * 0.5, naa2)

def test_leakage(data):
    """
    Test setting leakage before and after burst search
    """
    # burst search, then set leakage
    data.burst_search()
    data.leakage = 0.04
    na1 = list(data.na)
    # set leakage, then burst search
    data.burst_search()
    na2 = list(data.na)
    assert list_array_equal(na1, na2)

def test_gamma(data):
    """
    Test setting gamma before and after burst search
    """
    # burst search, then set gamma
    data.burst_search()
    E0 = list(data.E)
    data.gamma = 0.5
    E1 = list(data.E)
    assert not list_array_equal(E0, E1)
    # burst search after setting gamma
    data.burst_search()
    E2 = list(data.E)
    assert list_array_equal(E1, E2)

def test_dir_ex(data_1ch):
    """
    Test setting dir_ex before and after burst search
    """
    data = data_1ch
    # burst search, then set dir_ex
    data.burst_search()
    na0 = list(data.na)
    data.dir_ex = 0.05
    na1 = list(data.na)
    assert not list_array_equal(na0, na1)
    # burst search after setting dir_ex
    data.burst_search()
    na2 = list(data.na)
    assert list_array_equal(na1, na2)

def test_beta(data_1ch):
    """
    Test setting beta before and after burst search
    """
    data = data_1ch
    # burst search, then set beta
    data.burst_search()
    S0 = list(data.S)
    data.beta = 0.7
    S1 = list(data.S)
    assert not list_array_equal(S0, S1)
    # burst search after setting beta
    data.burst_search()
    S2 = list(data.S)
    assert list_array_equal(S1, S2)

def test_bursts_interface(data):
    d = data
    for b in d.mburst:
        assert (b.start == b.data[:, b._i_start]).all()
        assert (b.stop == b.data[:, b._i_stop]).all()
        assert (b.istart == b.data[:, b._i_istart]).all()
        assert (b.istop == b.data[:, b._i_istop]).all()

        rate = 1.*b.counts/b.width
        assert (b.ph_rate == rate).all()

        separation = b.start[1:] - b.stop[:-1]
        assert (b.separation == separation).all()

        assert (b.stop > b.start).all()

def test_burst_stop_istop(data):
    """Test coherence between b_end() and b_iend()"""
    d = data
    for ph, bursts in zip(d.ph_times_m, d.mburst):
        assert (ph[bursts.istop] == bursts.stop).all()

def test_monotonic_burst_start(data):
    """Test for monotonic burst start times."""
    d = data
    for i in range(d.nch):
        assert (np.diff(d.mburst[i].start) > 0).all()

def test_monotonic_burst_stop(data):
    """Test for monotonic burst stop times."""
    d = data
    for bursts in d.mburst:
        assert (np.diff(bursts.stop) > 0).all()

def test_burst_istart_iend_size(data):
    """Test consistency between burst istart, istop and counts (i.e. size)"""
    d = data
    for bursts in d.mburst:
        counts = bursts.istop - bursts.istart + 1
        assert (counts == bursts.counts).all()

def test_burst_recompute_times(data):
    """Test Bursts.recompute_times method."""
    d = data
    for times, bursts in zip(d.ph_times_m, d.mburst):
        newbursts = bursts.recompute_times(times)
        assert newbursts == bursts

def test_burst_recompute_index(data):
    """Test Bursts.recompute_index_* methods."""
    d = data
    ph_sel = Ph_sel(Dex='Dem')
    d.burst_search(ph_sel=ph_sel, index_allph=True)
    d_sel = d.copy()
    d_sel.burst_search(ph_sel=ph_sel, index_allph=False)
    for times_sel, mask_sel, bursts_sel, times_allph, bursts_allph in zip(
            d.iter_ph_times(ph_sel=ph_sel),
            d.iter_ph_masks(ph_sel=ph_sel),
            d_sel.mburst,
            d.iter_ph_times(),
            d.mburst):
        assert (times_sel[bursts_sel.istart] == bursts_sel.start).all()
        assert (times_sel[bursts_sel.istop] == bursts_sel.stop).all()

        assert (times_allph[bursts_allph.istart] == bursts_allph.start).all()
        assert (times_allph[bursts_allph.istop] == bursts_allph.stop).all()

        # Test individual methods
        bursts_allph2 = bursts_sel.recompute_index_expand(mask_sel)
        assert  bursts_allph2 == bursts_allph
        assert (times_allph[bursts_allph2.istart] == bursts_allph2.start).all()
        assert (times_allph[bursts_allph2.istop] == bursts_allph2.stop).all()

        bursts_sel2 = bursts_allph.recompute_index_reduce(times_sel)
        assert (times_sel[bursts_sel2.istart] == bursts_sel2.start).all()
        assert (times_sel[bursts_sel2.istop] == bursts_sel2.stop).all()
        assert  bursts_sel2 == bursts_sel

        # Test round-trip
        bursts_allph3 = bursts_sel2.recompute_index_expand(mask_sel)
        assert  bursts_allph3 == bursts_allph2
        assert (times_allph[bursts_allph3.istart] == bursts_allph3.start).all()
        assert (times_allph[bursts_allph3.istop] == bursts_allph3.stop).all()

## This test is only used to develop alternative implementations of
## Bursts.recompute_index_reduce() and is normally disabled as it is very slow.
#def test_burst_recompute_index_reduce(data):
#    """Test different versions of Bursts.recompute_index_reduce methods.
#
#    This test is very slow so it's normally disabled.
#    """
#    d = data
#    ph_sel = Ph_sel(Dex='Aem')
#    d.burst_search(ph_sel=ph_sel)
#    d_sel = d.copy()
#    d_sel.burst_search(ph_sel=ph_sel, index_allph=False)
#    for times_sel, bursts_sel, times_allph, bursts_allph in zip(
#            d.iter_ph_times(ph_sel=ph_sel),
#            d_sel.mburst,
#            d.iter_ph_times(),
#            d.mburst):
#        assert (times_allph[bursts_allph.istart] == bursts_allph.start).all()
#        assert (times_allph[bursts_allph.istop] == bursts_allph.stop).all()
#
#        bursts_sel1 = bursts_allph.recompute_index_reduce(times_sel)
#        bursts_sel2 = bursts_allph.recompute_index_reduce2(times_sel)
#        assert  bursts_sel1 == bursts_sel2
#        assert  bursts_sel == bursts_sel1

def test_phrates_mtuple(data):
    d = data
    m = 10
    max_num_ph = 20001
    for ph in d.iter_ph_times():
        phc = ph[:max_num_ph]
        rates = phrates.mtuple_rates(phc, m)
        delays = phrates.mtuple_delays(phc, m)
        t_rates = 0.5 * (phc[m-1:] + phc[:-m+1])
        assert phrates.mtuple_rates_max(phc, m) == rates.max()
        assert phrates.mtuple_delays_min(phc, m) == delays.min()
        assert phrates.default_c == 1
        assert (rates == (m - 1 - phrates.default_c) / delays).all()
        assert (phrates.mtuple_rates_t(phc, m) == t_rates).all()

if has_numba:
    def test_phrates_kde(data):
        d = data
        tau = 5000  # 5000 * 12.5ns = 6.25 us
        for ph in d.iter_ph_times():
            # Test consistency of kde_laplace_nph and (kde_laplace, kde_rect)
            rates = phrates.kde_laplace(ph, tau)
            nrect = phrates.kde_rect(ph, tau*10)
            ratesl, nph = phrates.nb.kde_laplace_nph(ph, tau)
            assert (rates == ratesl).all()
            assert (nph == nrect).all()

            # Test consistency of kde_laplace and _kde_laplace_self_numba
            ratesl2, nph2 = phrates.nb.kde_laplace_self_numba(ph, tau)
            assert (nph2 == nrect).all()
            assert (ratesl2 == rates).all()

            # Smoke test laplace, gaussian, rect with time_axis
            ratesl = phrates.kde_laplace(ph, tau, time_axis=ph+1)
            assert ((ratesl >= 0) * (ratesl < 5e6)).all()
            ratesg = phrates.kde_gaussian(ph, tau, time_axis=ph+1)
            assert ((ratesg >= 0) * (ratesg < 5e6)).all()
            ratesr = phrates.kde_rect(ph, tau, time_axis=ph+1)
            assert ((ratesr >= 0) * (ratesr < 5e6)).all()

    def test_phrates_kde_cy(data):
        d = data
        tau = 5000  # 5000 * 12.5ns = 6.25 us
        for ph in d.iter_ph_times():
            # Test consistency of kde_laplace_nph and (kde_laplace, kde_rect)
            ratesg = phrates.nb.kde_gaussian_numba(ph, tau)
            ratesl = phrates.nb.kde_laplace_numba(ph, tau)
            ratesr = phrates.nb.kde_rect_numba(ph, tau)
            ratesgc = phrates.cy.kde_gaussian_cy(ph, tau)
            rateslc = phrates.cy.kde_laplace_cy(ph, tau)
            ratesrc = phrates.cy.kde_rect_cy(ph, tau)
            assert (ratesg == ratesgc).all()
            assert (ratesl == rateslc).all()
            assert (ratesr == ratesrc).all()

def test_burst_ph_data_functions(data):
    """Tests the functions that iterate or operate on per-burst "ph-data".
    """
    d = data
    for bursts, ph, mask in zip(d.mburst, d.iter_ph_times(),
                                d.iter_ph_masks(Ph_sel(Dex='Dem'))):
        bstart = bursts.start
        bend = bursts.stop

        for i, (start, stop) in enumerate(bl.iter_bursts_start_stop(bursts)):
            assert ph[start] == bstart[i]
            assert ph[stop-1] == bend[i]

        for i, burst_ph in enumerate(bl.iter_bursts_ph(ph, bursts)):
            assert burst_ph[0] == bstart[i]
            assert burst_ph[-1] == bend[i]

        for i, burst_ph in enumerate(bl.iter_bursts_ph(ph, bursts, mask=mask)):
            if burst_ph.size > 0:
                assert burst_ph[0] >= bstart[i]
                assert burst_ph[-1] <= bend[i]

        stats = bl.burst_ph_stats(ph, bursts, mask=mask)
        assert (stats[~np.isnan(stats)] >= bstart[~np.isnan(stats)]).all()
        assert (stats[~np.isnan(stats)] <= bend[~np.isnan(stats)]).all()

        bistart = bursts.istart
        biend = bursts.istop
        bursts_mask = bl.ph_in_bursts_mask(ph.size, bursts)
        for i, (start, stop) in enumerate(bl.iter_bursts_start_stop(bursts)):
            assert bursts_mask[start:stop].all()
            if start > 0:
                if i > 0 and biend[i-1] < bistart[i] - 1:
                    assert not bursts_mask[start - 1]
            if stop < ph.size:
                if i < bistart.size-1 and bistart[i+1] > biend[i] + 1:
                    assert not bursts_mask[stop]

def test_ph_in_bursts_ich(data):
    """Tests the ph_in_bursts_ich method.
    """
    d = data
    for ich in range(d.nch):
        ph_in_bursts = d.ph_in_bursts_ich(ich)
        ph_in_bursts_dd = d.ph_in_bursts_ich(ich, ph_sel=Ph_sel(Dex='Dem'))
        assert ph_in_bursts_dd.size < ph_in_bursts.size

def test_burst_fuse(data):
    """Test 2 independent implementations of fuse_bursts for consistency.
    """
    d = data
    for bursts in d.mburst:
        new_mbursti = bl.fuse_bursts_iter(bursts, ms=1)
        new_mburstd = bl.fuse_bursts_direct(bursts, ms=1)
        assert new_mbursti == new_mburstd

def test_burst_fuse_0ms(data):
    """Test that after fusing with ms=0 the sum of bursts sizes is that same
    as the number of ph in bursts (via burst selection).
    """
    d = data
    if d.nch == 8:
        d.burst_search(L=10, m=10, F=7, computefret=False)
        d.mburst[1] = bl.bslib.Bursts.empty()  # Make one channel with no bursts
        d._calc_burst_period()
        d.calc_fret(count_ph=True)
    df = d.fuse_bursts(ms=0)
    for ich, bursts in enumerate(df.mburst):
        mask = bl.ph_in_bursts_mask(df.ph_data_sizes[ich], bursts)
        assert mask.sum() == bursts.counts.sum()
    df.calc_fret(count_ph=True)
    assert len(df.mburst) == len(d.mburst)
    assert len(df.mburst) == d.nch

def test_burst_fuse_separation(data):
    """Test that after fusing bursts the minimum separation is equal
    to the threshold used during fusing.
    """
    d = data
    fuse_ms = 2
    df = d.fuse_bursts(ms=fuse_ms)
    for bursts in df.mburst:
        separation = bursts.separation * df.clk_p
        if bursts.num_bursts > 0:
            assert separation.min() >= fuse_ms * 1e-3

def test_calc_sbr(data):
    """Smoke test Data.calc_sbr()"""
    data.calc_sbr()

def test_calc_max_rate(data):
    """Smoke test for Data.calc_max_rate()"""
    data.calc_max_rate(m=10)
    if data.alternated:
        data.calc_max_rate(m=10, ph_sel=Ph_sel(Dex='DAem'), compact=True)

def test_burst_data(data):
    """Test for bext.burst_data()"""
    bext.burst_data(data, include_bg=True, include_ph_index=True)
    bext.burst_data(data, include_bg=False, include_ph_index=True)
    bext.burst_data(data, include_bg=True, include_ph_index=False)
    bext.burst_data(data, include_bg=False, include_ph_index=False)

def test_print_burst_stats(data):
    """Smoke test for burstlib.print_burst_stats()"""
    bl.print_burst_stats(data)

def test_expand(data):
    """Test method `expand()` for `Data()`."""
    d = data
    for ich, bursts in enumerate(d.mburst):
        if bursts.num_bursts == 0:
            continue  # if no bursts skip this ch
        nd, na, bg_d, bg_a, width = d.expand(ich, width=True)
        width2 = bursts.width * d.clk_p
        period = d.bp[ich]
        bg_d2 = d.bg_from(Ph_sel(Dex='Dem'))[ich][period] * width2
        bg_a2 = d.bg_from(Ph_sel(Dex='Aem'))[ich][period] * width2
        assert (width == width2).all()
        assert (nd == d.nd[ich]).all() and (na == d.na[ich]).all()
        assert (bg_d == bg_d2).all() and (bg_a == bg_a2).all()

def test_burst_data_ich(data):
    """Test method `Data.burst_data_ich()`."""
    d = data
    for ich, bursts in enumerate(d.mburst):
        if bursts.num_bursts == 0:
            continue  # if no bursts skip this ch
        burst_dict = d.burst_data_ich(ich=ich)
        assert (burst_dict['size_raw'] == bursts.counts).all()
        assert (burst_dict['t_start'] == bursts.start * d.clk_p).all()
        assert (burst_dict['t_stop'] == bursts.stop * d.clk_p).all()
        assert (burst_dict['i_start'] == bursts.istart).all()
        assert (burst_dict['i_stop'] == bursts.istop).all()
        assert (burst_dict['bg_period'] == d.bp[ich]).all()
        nd, na, bg_d, bg_a, width = d.expand(ich, width=True)
        width_ms = width * 1e3
        assert (width_ms == burst_dict['width_ms']).all()
        assert (nd == burst_dict['nd']).all()
        assert (na == burst_dict['na']).all()
        assert (bg_d == burst_dict['bg_dd']).all()
        assert (bg_a == burst_dict['bg_ad']).all()
        if d.alternated:
            period = d.bp[ich]
            bg_da = d.bg_from(Ph_sel(Aex='Dem'))[ich][period] * width
            bg_aa = d.bg_from(Ph_sel(Aex='Aem'))[ich][period] * width
            assert (bg_da == burst_dict['bg_da']).all()
            assert (bg_aa == burst_dict['bg_aa']).all()

def test_burst_corrections(data):
    """Test background and bleed-through corrections."""
    d = data
    d.calc_ph_num(alex_all=True)
    d.corrections()
    leakage = d.get_leakage_array()

    for ich, bursts in enumerate(d.mburst):
        if bursts.num_bursts == 0: continue  # if no bursts skip this ch
        nd, na, bg_d, bg_a, width = d.expand(ich, width=True)
        burst_size_raw = bursts.counts

        lk = leakage[ich]
        if d.alternated:
            nda, naa = d.nda[ich], d.naa[ich]
            period = d.bp[ich]
            bg_da = d.bg_from(Ph_sel(Aex='Dem'))[ich][period]*width
            bg_aa = d.bg_from(Ph_sel(Aex='Aem'))[ich][period]*width
            burst_size_raw2 = (nd + na + bg_d + bg_a + lk*nd + nda + naa +
                               bg_da + bg_aa)
            assert np.allclose(burst_size_raw, burst_size_raw2)
        else:
            burst_size_raw2 = nd + na + bg_d + bg_a + lk*nd
            assert np.allclose(burst_size_raw, burst_size_raw2)

def test_burst_search_consistency(data):
    """Test consistency of burst data array
    """
    d = data
    for mb, ph in zip(d.mburst, d.iter_ph_times()):
        tot_size = mb.counts
        istart, istop = mb.istart, mb.istop
        assert np.all(tot_size == istop - istart + 1)
        start, stop, width = mb.start, mb.stop, mb.width
        assert np.all(width == stop - start)
    df = d.fuse_bursts(ms=0)
    for mb, ph in zip(df.mburst, df.iter_ph_times()):
        tot_size = mb.counts
        istart, istop = mb.istart, mb.istop
        assert np.all(tot_size == istop - istart + 1)
        start, stop, width = mb.start, mb.stop, mb.width
        assert np.all(width == stop - start)
    df = d.fuse_bursts(ms=1)
    for mb, ph in zip(df.mburst, df.iter_ph_times()):
        tot_size = mb.counts
        istart, istop = mb.istart, mb.istop
        assert np.all(tot_size <= istop - istart + 1)
        start, stop, width = mb.start, mb.stop, mb.width
        assert np.all(width <= stop - start)

def test_E_and_S_with_corrections(data):
    d = data
    gamma = 0.5
    beta = 0.7
    d.gamma = gamma
    d.beta = beta
    for i, (E, nd, na) in enumerate(zip(d.E, d.nd, d.na)):
        assert (E == na / (nd * gamma + na)).all()
        if d.alternated:
            naa = d.naa[i]
            if 'PAX' in data.meas_type:
                naa = d.naa[i] - d.nar[i]
            assert (d.S[i] == (gamma * nd + na) /
                              (gamma * nd + na + naa / beta)).all()

def test_burst_size_da(data):
    """Test that nd + na with no corrections is equal to b_size(mburst).
    """
    d = data
    d.calc_ph_num(alex_all=True)
    if d.alternated:
        for mb, nd, na, naa, nda in zip(d.mburst, d.nd, d.na, d.naa, d.nda):
            tot_size = mb.counts
            tot_size2 = nd + na + naa + nda
            assert np.allclose(tot_size, tot_size2)
    else:
        for mb, nd, na in zip(d.mburst, d.nd, d.na):
            tot_size = mb.counts
            assert (tot_size == nd + na).all()

def test_burst_selection(data):
    """Smoke test for burst selection methods.
    """
    d = data
    d.select_bursts(select_bursts.size, th1=20, th2=100, add_naa=True)
    d.select_bursts(select_bursts.size, th1=20, th2=100, gamma=0.5)

    M1 = d.select_bursts_mask(select_bursts.consecutive, th1=1e-3, th2=1e4,
                              kind='first')
    M2 = d.select_bursts_mask(select_bursts.consecutive, th1=1e-3, th2=1e4,
                              kind='second')
    Mb = d.select_bursts_mask(select_bursts.consecutive, th1=1e-3, th2=1e4,
                              kind='both')
    Mb2 = [m1 + m2 for m1, m2 in zip(M1, M2)]
    assert list_array_equal(Mb, Mb2)

def test_burst_selection_nocorrections(data):
    """Test burst selection with uncorrected bursts.
    """
    d = data
    d.burst_search(computefret=False)
    d.calc_fret(count_ph=True, corrections=False)
    ds1 = d.select_bursts(select_bursts.size, th1=20, th2=100,
                          computefret=False)
    ds2 = d.select_bursts(select_bursts.size, th1=20, th2=100)
    ds2.calc_ph_num()
    ds2.calc_fret(corrections=False)

    assert list_array_equal(ds1.nd, ds2.nd)
    assert list_array_equal(ds1.na, ds2.na)
    assert list_array_equal(ds1.E, ds2.E)
    if d.alternated:
        assert list_array_equal(ds1.naa, ds2.naa)
        assert list_array_equal(ds1.E, ds2.E)

def test_burst_selection_ranges(data):
    """Test selection functions having a min-max range.
    """
    d = data
    d.burst_search()
    d.calc_max_rate(m=10, ph_sel=Ph_sel(Dex='DAem'))

    Range = namedtuple('Range', ['min', 'max', 'getter'])

    sel_functions = dict(
        E=Range(0.5, 1, None), nd=Range(30, 40, None), na=Range(30, 40, None),
        time=Range(1, 61, lambda d, ich: d.mburst[ich].start * d.clk_p),
        width=Range(0.5, 1.5, lambda d, ich: d.mburst[ich].width * d.clk_p*1e3),
        peak_phrate=Range(50e3, 150e3, lambda d, ich: d.max_rate[ich]))
    if d.alternated:
        sel_functions.update(naa=Range(30, 40, None), S=Range(0.3, 0.7, None))

    for func_name, range_ in sel_functions.items():
        func = getattr(select_bursts, func_name)
        getter = range_.getter
        if getter is None:
            getter = lambda d, ich: d[func_name][ich]

        ds = d.select_bursts(func, args=(range_.min, range_.max))
        for ich in range(d.nch):
            selected = getter(ds, ich)
            assert ((selected >= range_.min) * (selected <= range_.max)).all()

def test_join_data(data):
    """Smoke test for bext.join_data() function.
    """
    d = data
    dj = bext.join_data([d, d.copy()])
    assert (dj.num_bursts == 2 * d.num_bursts).all()
    for bursts in dj.mburst:
        assert (np.diff(bursts.start) > 0).all()

def test_collapse(data_8ch):
    """Test the .collapse() method that joins the ch.
    """
    d = data_8ch
    dc1 = d.collapse()
    bursts1 = dc1.mburst[0]
    bursts2 = bl.bslib.Bursts.merge(d.mburst, sort=True)
    assert bursts1 == bursts2
    bursts2 = bl.bslib.Bursts.merge(d.mburst, sort=False)
    indexsort_stop = bursts2.stop.argsort()
    bursts3 = bursts2[indexsort_stop]
    indexsort_start = bursts3.start.argsort()
    bursts4 = bursts3[indexsort_start]
    assert bursts1 == bursts4
    indexsort = np.lexsort((bursts2.stop, bursts2.start))
    for name in d.burst_fields:
        if name not in d or name == 'mburst':
            continue
        newfield = np.hstack(d[name])[indexsort]
        assert np.allclose(dc1[name][0], newfield)

    dc2 = d.collapse(update_gamma=False)
    for name in d.burst_fields:
        if name not in d: continue

        if name == 'mburst':
            assert dc1.mburst[0] == dc2.mburst[0]
        else:
            assert np.allclose(dc1[name][0], dc2[name][0])

if __name__ == '__main__':
    pytest.main("-x -v fretbursts/tests/test_burstlib.py")


from __future__ import print_function, division
from time import time
import argparse
import numpy as np

from sklearn.dummy import DummyClassifier

from sklearn.datasets import fetch_20newsgroups_vectorized
from sklearn.metrics import accuracy_score
from sklearn.utils.validation import check_array

from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB

ESTIMATORS = {
    "dummy": DummyClassifier(),
    "random_forest": RandomForestClassifier(n_estimators=100,
                                            max_features="sqrt",
                                            min_samples_split=10),
    "extra_trees": ExtraTreesClassifier(n_estimators=100,
                                        max_features="sqrt",
                                        min_samples_split=10),
    "logistic_regression": LogisticRegression(),
    "naive_bayes": MultinomialNB(),
    "adaboost": AdaBoostClassifier(n_estimators=10),
}

###############################################################################
# Data

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('-e', '--estimators', nargs="+", required=True,
                        choices=ESTIMATORS)
    args = vars(parser.parse_args())

    data_train = fetch_20newsgroups_vectorized(subset="train")
    data_test = fetch_20newsgroups_vectorized(subset="test")
    X_train = check_array(data_train.data, dtype=np.float32,
                          accept_sparse="csc")
    X_test = check_array(data_test.data, dtype=np.float32, accept_sparse="csr")
    y_train = data_train.target
    y_test = data_test.target

    print("20 newsgroups")
    print("=============")
    print("X_train.shape = {0}".format(X_train.shape))
    print("X_train.format = {0}".format(X_train.format))
    print("X_train.dtype = {0}".format(X_train.dtype))
    print("X_train density = {0}"
          "".format(X_train.nnz / np.product(X_train.shape)))
    print("y_train {0}".format(y_train.shape))
    print("X_test {0}".format(X_test.shape))
    print("X_test.format = {0}".format(X_test.format))
    print("X_test.dtype = {0}".format(X_test.dtype))
    print("y_test {0}".format(y_test.shape))
    print()

    print("Classifier Training")
    print("===================")
    accuracy, train_time, test_time = {}, {}, {}
    for name in sorted(args["estimators"]):
        clf = ESTIMATORS[name]
        try:
            clf.set_params(random_state=0)
        except (TypeError, ValueError):
            pass

        print("Training %s ... " % name, end="")
        t0 = time()
        clf.fit(X_train, y_train)
        train_time[name] = time() - t0
        t0 = time()
        y_pred = clf.predict(X_test)
        test_time[name] = time() - t0
        accuracy[name] = accuracy_score(y_test, y_pred)
        print("done")

    print()
    print("Classification performance:")
    print("===========================")
    print()
    print("%s %s %s %s" % ("Classifier  ", "train-time", "test-time",
                           "Accuracy"))
    print("-" * 44)
    for name in sorted(accuracy, key=accuracy.get):
        print("%s %s %s %s" % (name.ljust(16),
                               ("%.4fs" % train_time[name]).center(10),
                               ("%.4fs" % test_time[name]).center(10),
                               ("%.4f" % accuracy[name]).center(10)))

    print()


from collections import OrderedDict

import pandas as pd

from bokeh.charts import TimeSeries, show, output_file

# read in some stock data from the Yahoo Finance API
AAPL = pd.read_csv(
    "http://ichart.yahoo.com/table.csv?s=AAPL&a=0&b=1&c=2000&d=0&e=1&f=2010",
    parse_dates=['Date'])
MSFT = pd.read_csv(
    "http://ichart.yahoo.com/table.csv?s=MSFT&a=0&b=1&c=2000&d=0&e=1&f=2010",
    parse_dates=['Date'])
IBM = pd.read_csv(
    "http://ichart.yahoo.com/table.csv?s=IBM&a=0&b=1&c=2000&d=0&e=1&f=2010",
    parse_dates=['Date'])

xyvalues = OrderedDict(
    AAPL=AAPL['Adj Close'],
    Date=AAPL['Date'],
    MSFT=MSFT['Adj Close'],
    IBM=IBM['Adj Close'],
)

# any of the following commented are valid Bar inputs
#xyvalues = pd.DataFrame(xyvalues)
#lindex = xyvalues.pop('Date')
#lxyvalues = list(xyvalues.values())
#lxyvalues = np.array(xyvalues.values())

TOOLS="resize,pan,wheel_zoom,box_zoom,reset,previewsave"

output_file("stocks_timeseries.html")

ts = TimeSeries(
    xyvalues, index='Date', legend=True,
    title="Timeseries", tools=TOOLS, ylabel='Stock Prices')

# usage with iterable index
#ts = TimeSeries(
#    lxyvalues, index=lindex,
#    title="timeseries, pd_input", ylabel='Stock Prices')

show(ts)