from __future__ import division, print_function, absolute_import

import numpy as np
from scipy._lib.six import callable, xrange
from scipy._lib._numpy_compat import suppress_warnings
from collections import namedtuple

__all__ = ['binned_statistic',
           'binned_statistic_2d',
           'binned_statistic_dd']

BinnedStatisticResult = namedtuple('BinnedStatisticResult',
                                   ('statistic', 'bin_edges', 'binnumber'))

def binned_statistic(x, values, statistic='mean',
                     bins=10, range=None):
    """
    Compute a binned statistic for one or more sets of data.

    This is a generalization of a histogram function.  A histogram divides
    the space into bins, and returns the count of the number of points in
    each bin.  This function allows the computation of the sum, mean, median,
    or other statistic of the values (or set of values) within each bin.

    Parameters
    ----------
    x : (N,) array_like
        A sequence of values to be binned.
    values : (N,) array_like or list of (N,) array_like
        The data on which the statistic will be computed.  This must be
        the same shape as `x`, or a set of sequences - each the same shape as
        `x`.  If `values` is a set of sequences, the statistic will be computed
        on each independently.
    statistic : string or callable, optional
        The statistic to compute (default is 'mean').
        The following statistics are available:

          * 'mean' : compute the mean of values for points within each bin.
            Empty bins will be represented by NaN.
          * 'median' : compute the median of values for points within each
            bin. Empty bins will be represented by NaN.
          * 'count' : compute the count of points within each bin.  This is
            identical to an unweighted histogram.  `values` array is not
            referenced.
          * 'sum' : compute the sum of values for points within each bin.
            This is identical to a weighted histogram.
          * 'min' : compute the minimum of values for points within each bin.
            Empty bins will be represented by NaN.
          * 'max' : compute the maximum of values for point within each bin.
            Empty bins will be represented by NaN.
          * function : a user-defined function which takes a 1D array of
            values, and outputs a single numerical statistic. This function
            will be called on the values in each bin.  Empty bins will be
            represented by function([]), or NaN if this returns an error.

    bins : int or sequence of scalars, optional
        If `bins` is an int, it defines the number of equal-width bins in the
        given range (10 by default).  If `bins` is a sequence, it defines the
        bin edges, including the rightmost edge, allowing for non-uniform bin
        widths.  Values in `x` that are smaller than lowest bin edge are
        assigned to bin number 0, values beyond the highest bin are assigned to
        ``bins[-1]``.  If the bin edges are specified, the number of bins will
        be, (nx = len(bins)-1).
    range : (float, float) or [(float, float)], optional
        The lower and upper range of the bins.  If not provided, range
        is simply ``(x.min(), x.max())``.  Values outside the range are
        ignored.

    Returns
    -------
    statistic : array
        The values of the selected statistic in each bin.
    bin_edges : array of dtype float
        Return the bin edges ``(length(statistic)+1)``.
    binnumber: 1-D ndarray of ints
        Indices of the bins (corresponding to `bin_edges`) in which each value
        of `x` belongs.  Same length as `values`.  A binnumber of `i` means the
        corresponding value is between (bin_edges[i-1], bin_edges[i]).

    See Also
    --------
    numpy.digitize, numpy.histogram, binned_statistic_2d, binned_statistic_dd

    Notes
    -----
    All but the last (righthand-most) bin is half-open.  In other words, if
    `bins` is ``[1, 2, 3, 4]``, then the first bin is ``[1, 2)`` (including 1,
    but excluding 2) and the second ``[2, 3)``.  The last bin, however, is
    ``[3, 4]``, which *includes* 4.

    .. versionadded:: 0.11.0

    Examples
    --------
    >>> from scipy import stats
    >>> import matplotlib.pyplot as plt

    First some basic examples:

    Create two evenly spaced bins in the range of the given sample, and sum the
    corresponding values in each of those bins:

    >>> values = [1.0, 1.0, 2.0, 1.5, 3.0]
    >>> stats.binned_statistic([1, 1, 2, 5, 7], values, 'sum', bins=2)
    (array([ 4. ,  4.5]), array([ 1.,  4.,  7.]), array([1, 1, 1, 2, 2]))

    Multiple arrays of values can also be passed.  The statistic is calculated
    on each set independently:

    >>> values = [[1.0, 1.0, 2.0, 1.5, 3.0], [2.0, 2.0, 4.0, 3.0, 6.0]]
    >>> stats.binned_statistic([1, 1, 2, 5, 7], values, 'sum', bins=2)
    (array([[ 4. ,  4.5], [ 8. ,  9. ]]), array([ 1.,  4.,  7.]),
        array([1, 1, 1, 2, 2]))

    >>> stats.binned_statistic([1, 2, 1, 2, 4], np.arange(5), statistic='mean',
    ...                        bins=3)
    (array([ 1.,  2.,  4.]), array([ 1.,  2.,  3.,  4.]),
        array([1, 2, 1, 2, 3]))

    As a second example, we now generate some random data of sailing boat speed
    as a function of wind speed, and then determine how fast our boat is for
    certain wind speeds:

    >>> windspeed = 8 * np.random.rand(500)
    >>> boatspeed = .3 * windspeed**.5 + .2 * np.random.rand(500)
    >>> bin_means, bin_edges, binnumber = stats.binned_statistic(windspeed,
    ...                 boatspeed, statistic='median', bins=[1,2,3,4,5,6,7])
    >>> plt.figure()
    >>> plt.plot(windspeed, boatspeed, 'b.', label='raw data')
    >>> plt.hlines(bin_means, bin_edges[:-1], bin_edges[1:], colors='g', lw=5,
    ...            label='binned statistic of data')
    >>> plt.legend()

    Now we can use ``binnumber`` to select all datapoints with a windspeed
    below 1:

    >>> low_boatspeed = boatspeed[binnumber == 0]

    As a final example, we will use ``bin_edges`` and ``binnumber`` to make a
    plot of a distribution that shows the mean and distribution around that
    mean per bin, on top of a regular histogram and the probability
    distribution function:

    >>> x = np.linspace(0, 5, num=500)
    >>> x_pdf = stats.maxwell.pdf(x)
    >>> samples = stats.maxwell.rvs(size=10000)

    >>> bin_means, bin_edges, binnumber = stats.binned_statistic(x, x_pdf,
    ...         statistic='mean', bins=25)
    >>> bin_width = (bin_edges[1] - bin_edges[0])
    >>> bin_centers = bin_edges[1:] - bin_width/2

    >>> plt.figure()
    >>> plt.hist(samples, bins=50, normed=True, histtype='stepfilled',
    ...          alpha=0.2, label='histogram of data')
    >>> plt.plot(x, x_pdf, 'r-', label='analytical pdf')
    >>> plt.hlines(bin_means, bin_edges[:-1], bin_edges[1:], colors='g', lw=2,
    ...            label='binned statistic of data')
    >>> plt.plot((binnumber - 0.5) * bin_width, x_pdf, 'g.', alpha=0.5)
    >>> plt.legend(fontsize=10)
    >>> plt.show()

    """
    try:
        N = len(bins)
    except TypeError:
        N = 1

    if N != 1:
        bins = [np.asarray(bins, float)]

    if range is not None:
        if len(range) == 2:
            range = [range]

    medians, edges, binnumbers = binned_statistic_dd(
        [x], values, statistic, bins, range)

    return BinnedStatisticResult(medians, edges[0], binnumbers)

BinnedStatistic2dResult = namedtuple('BinnedStatistic2dResult',
                                     ('statistic', 'x_edge', 'y_edge',
                                      'binnumber'))

def binned_statistic_2d(x, y, values, statistic='mean',
                        bins=10, range=None, expand_binnumbers=False):
    """
    Compute a bidimensional binned statistic for one or more sets of data.

    This is a generalization of a histogram2d function.  A histogram divides
    the space into bins, and returns the count of the number of points in
    each bin.  This function allows the computation of the sum, mean, median,
    or other statistic of the values (or set of values) within each bin.

    Parameters
    ----------
    x : (N,) array_like
        A sequence of values to be binned along the first dimension.
    y : (N,) array_like
        A sequence of values to be binned along the second dimension.
    values : (N,) array_like or list of (N,) array_like
        The data on which the statistic will be computed.  This must be
        the same shape as `x`, or a list of sequences - each with the same
        shape as `x`.  If `values` is such a list, the statistic will be
        computed on each independently.
    statistic : string or callable, optional
        The statistic to compute (default is 'mean').
        The following statistics are available:

          * 'mean' : compute the mean of values for points within each bin.
            Empty bins will be represented by NaN.
          * 'median' : compute the median of values for points within each
            bin. Empty bins will be represented by NaN.
          * 'count' : compute the count of points within each bin.  This is
            identical to an unweighted histogram.  `values` array is not
            referenced.
          * 'sum' : compute the sum of values for points within each bin.
            This is identical to a weighted histogram.
          * 'min' : compute the minimum of values for points within each bin.
            Empty bins will be represented by NaN.
          * 'max' : compute the maximum of values for point within each bin.
            Empty bins will be represented by NaN.
          * function : a user-defined function which takes a 1D array of
            values, and outputs a single numerical statistic. This function
            will be called on the values in each bin.  Empty bins will be
            represented by function([]), or NaN if this returns an error.

    bins : int or [int, int] or array_like or [array, array], optional
        The bin specification:

          * the number of bins for the two dimensions (nx = ny = bins),
          * the number of bins in each dimension (nx, ny = bins),
          * the bin edges for the two dimensions (x_edge = y_edge = bins),
          * the bin edges in each dimension (x_edge, y_edge = bins).

        If the bin edges are specified, the number of bins will be,
        (nx = len(x_edge)-1, ny = len(y_edge)-1).

    range : (2,2) array_like, optional
        The leftmost and rightmost edges of the bins along each dimension
        (if not specified explicitly in the `bins` parameters):
        [[xmin, xmax], [ymin, ymax]]. All values outside of this range will be
        considered outliers and not tallied in the histogram.
    expand_binnumbers : bool, optional
        'False' (default): the returned `binnumber` is a shape (N,) array of
        linearized bin indices.
        'True': the returned `binnumber` is 'unraveled' into a shape (2,N)
        ndarray, where each row gives the bin numbers in the corresponding
        dimension.
        See the `binnumber` returned value, and the `Examples` section.

        .. versionadded:: 0.17.0

    Returns
    -------
    statistic : (nx, ny) ndarray
        The values of the selected statistic in each two-dimensional bin.
    x_edge : (nx + 1) ndarray
        The bin edges along the first dimension.
    y_edge : (ny + 1) ndarray
        The bin edges along the second dimension.
    binnumber : (N,) array of ints or (2,N) ndarray of ints
        This assigns to each element of `sample` an integer that represents the
        bin in which this observation falls.  The representation depends on the
        `expand_binnumbers` argument.  See `Notes` for details.

    See Also
    --------
    numpy.digitize, numpy.histogram2d, binned_statistic, binned_statistic_dd

    Notes
    -----
    Binedges:
    All but the last (righthand-most) bin is half-open.  In other words, if
    `bins` is ``[1, 2, 3, 4]``, then the first bin is ``[1, 2)`` (including 1,
    but excluding 2) and the second ``[2, 3)``.  The last bin, however, is
    ``[3, 4]``, which *includes* 4.

    `binnumber`:
    This returned argument assigns to each element of `sample` an integer that
    represents the bin in which it belongs.  The representation depends on the
    `expand_binnumbers` argument. If 'False' (default): The returned
    `binnumber` is a shape (N,) array of linearized indices mapping each
    element of `sample` to its corresponding bin (using row-major ordering).
    If 'True': The returned `binnumber` is a shape (2,N) ndarray where
    each row indicates bin placements for each dimension respectively.  In each
    dimension, a binnumber of `i` means the corresponding value is between
    (D_edge[i-1], D_edge[i]), where 'D' is either 'x' or 'y'.

    .. versionadded:: 0.11.0

    Examples
    --------
    >>> from scipy import stats

    Calculate the counts with explicit bin-edges:

    >>> x = [0.1, 0.1, 0.1, 0.6]
    >>> y = [2.1, 2.6, 2.1, 2.1]
    >>> binx = [0.0, 0.5, 1.0]
    >>> biny = [2.0, 2.5, 3.0]
    >>> ret = stats.binned_statistic_2d(x, y, None, 'count', bins=[binx,biny])
    >>> ret.statistic
    array([[ 2.,  1.],
           [ 1.,  0.]])

    The bin in which each sample is placed is given by the `binnumber`
    returned parameter.  By default, these are the linearized bin indices:

    >>> ret.binnumber
    array([5, 6, 5, 9])

    The bin indices can also be expanded into separate entries for each
    dimension using the `expand_binnumbers` parameter:

    >>> ret = stats.binned_statistic_2d(x, y, None, 'count', bins=[binx,biny],
    ...                                 expand_binnumbers=True)
    >>> ret.binnumber
    array([[1, 1, 1, 2],
           [1, 2, 1, 1]])

    Which shows that the first three elements belong in the xbin 1, and the
    fourth into xbin 2; and so on for y.

    """

    # This code is based on np.histogram2d
    try:
        N = len(bins)
    except TypeError:
        N = 1

    if N != 1 and N != 2:
        xedges = yedges = np.asarray(bins, float)
        bins = [xedges, yedges]

    medians, edges, binnumbers = binned_statistic_dd(
        [x, y], values, statistic, bins, range,
        expand_binnumbers=expand_binnumbers)

    return BinnedStatistic2dResult(medians, edges[0], edges[1], binnumbers)

BinnedStatisticddResult = namedtuple('BinnedStatisticddResult',
                                     ('statistic', 'bin_edges',
                                      'binnumber'))

def binned_statistic_dd(sample, values, statistic='mean',
                        bins=10, range=None, expand_binnumbers=False):
    """
    Compute a multidimensional binned statistic for a set of data.

    This is a generalization of a histogramdd function.  A histogram divides
    the space into bins, and returns the count of the number of points in
    each bin.  This function allows the computation of the sum, mean, median,
    or other statistic of the values within each bin.

    Parameters
    ----------
    sample : array_like
        Data to histogram passed as a sequence of D arrays of length N, or
        as an (N,D) array.
    values : (N,) array_like or list of (N,) array_like
        The data on which the statistic will be computed.  This must be
        the same shape as `x`, or a list of sequences - each with the same
        shape as `x`.  If `values` is such a list, the statistic will be
        computed on each independently.
    statistic : string or callable, optional
        The statistic to compute (default is 'mean').
        The following statistics are available:

          * 'mean' : compute the mean of values for points within each bin.
            Empty bins will be represented by NaN.
          * 'median' : compute the median of values for points within each
            bin. Empty bins will be represented by NaN.
          * 'count' : compute the count of points within each bin.  This is
            identical to an unweighted histogram.  `values` array is not
            referenced.
          * 'sum' : compute the sum of values for points within each bin.
            This is identical to a weighted histogram.
          * 'min' : compute the minimum of values for points within each bin.
            Empty bins will be represented by NaN.
          * 'max' : compute the maximum of values for point within each bin.
            Empty bins will be represented by NaN.
          * function : a user-defined function which takes a 1D array of
            values, and outputs a single numerical statistic. This function
            will be called on the values in each bin.  Empty bins will be
            represented by function([]), or NaN if this returns an error.

    bins : sequence or int, optional
        The bin specification must be in one of the following forms:

          * A sequence of arrays describing the bin edges along each dimension.
          * The number of bins for each dimension (nx, ny, ... = bins).
          * The number of bins for all dimensions (nx = ny = ... = bins).

    range : sequence, optional
        A sequence of lower and upper bin edges to be used if the edges are
        not given explicitely in `bins`. Defaults to the minimum and maximum
        values along each dimension.
    expand_binnumbers : bool, optional
        'False' (default): the returned `binnumber` is a shape (N,) array of
        linearized bin indices.
        'True': the returned `binnumber` is 'unraveled' into a shape (D,N)
        ndarray, where each row gives the bin numbers in the corresponding
        dimension.
        See the `binnumber` returned value, and the `Examples` section of
        `binned_statistic_2d`.

        .. versionadded:: 0.17.0

    Returns
    -------
    statistic : ndarray, shape(nx1, nx2, nx3,...)
        The values of the selected statistic in each two-dimensional bin.
    bin_edges : list of ndarrays
        A list of D arrays describing the (nxi + 1) bin edges for each
        dimension.
    binnumber : (N,) array of ints or (D,N) ndarray of ints
        This assigns to each element of `sample` an integer that represents the
        bin in which this observation falls.  The representation depends on the
        `expand_binnumbers` argument.  See `Notes` for details.

    See Also
    --------
    numpy.digitize, numpy.histogramdd, binned_statistic, binned_statistic_2d

    Notes
    -----
    Binedges:
    All but the last (righthand-most) bin is half-open in each dimension.  In
    other words, if `bins` is ``[1, 2, 3, 4]``, then the first bin is
    ``[1, 2)`` (including 1, but excluding 2) and the second ``[2, 3)``.  The
    last bin, however, is ``[3, 4]``, which *includes* 4.

    `binnumber`:
    This returned argument assigns to each element of `sample` an integer that
    represents the bin in which it belongs.  The representation depends on the
    `expand_binnumbers` argument. If 'False' (default): The returned
    `binnumber` is a shape (N,) array of linearized indices mapping each
    element of `sample` to its corresponding bin (using row-major ordering).
    If 'True': The returned `binnumber` is a shape (D,N) ndarray where
    each row indicates bin placements for each dimension respectively.  In each
    dimension, a binnumber of `i` means the corresponding value is between
    (bin_edges[D][i-1], bin_edges[D][i]), for each dimension 'D'.

    .. versionadded:: 0.11.0

    """
    known_stats = ['mean', 'median', 'count', 'sum', 'std','min','max']
    if not callable(statistic) and statistic not in known_stats:
        raise ValueError('invalid statistic %r' % (statistic,))

    # `Ndim` is the number of dimensions (e.g. `2` for `binned_statistic_2d`)
    # `Dlen` is the length of elements along each dimension.
    # This code is based on np.histogramdd
    try:
        # `sample` is an ND-array.
        Dlen, Ndim = sample.shape
    except (AttributeError, ValueError):
        # `sample` is a sequence of 1D arrays.
        sample = np.atleast_2d(sample).T
        Dlen, Ndim = sample.shape

    # Store initial shape of `values` to preserve it in the output
    values = np.asarray(values)
    input_shape = list(values.shape)
    # Make sure that `values` is 2D to iterate over rows
    values = np.atleast_2d(values)
    Vdim, Vlen = values.shape

    # Make sure `values` match `sample`
    if(statistic != 'count' and Vlen != Dlen):
        raise AttributeError('The number of `values` elements must match the '
                             'length of each `sample` dimension.')

    nbin = np.empty(Ndim, int)    # Number of bins in each dimension
    edges = Ndim * [None]         # Bin edges for each dim (will be 2D array)
    dedges = Ndim * [None]        # Spacing between edges (will be 2D array)

    try:
        M = len(bins)
        if M != Ndim:
            raise AttributeError('The dimension of bins must be equal '
                                 'to the dimension of the sample x.')
    except TypeError:
        bins = Ndim * [bins]

    # Select range for each dimension
    # Used only if number of bins is given.
    if range is None:
        smin = np.atleast_1d(np.array(sample.min(axis=0), float))
        smax = np.atleast_1d(np.array(sample.max(axis=0), float))
    else:
        smin = np.zeros(Ndim)
        smax = np.zeros(Ndim)
        for i in xrange(Ndim):
            smin[i], smax[i] = range[i]

    # Make sure the bins have a finite width.
    for i in xrange(len(smin)):
        if smin[i] == smax[i]:
            smin[i] = smin[i] - .5
            smax[i] = smax[i] + .5

    # Create edge arrays
    for i in xrange(Ndim):
        if np.isscalar(bins[i]):
            nbin[i] = bins[i] + 2  # +2 for outlier bins
            edges[i] = np.linspace(smin[i], smax[i], nbin[i] - 1)
        else:
            edges[i] = np.asarray(bins[i], float)
            nbin[i] = len(edges[i]) + 1  # +1 for outlier bins
        dedges[i] = np.diff(edges[i])

    nbin = np.asarray(nbin)

    # Compute the bin number each sample falls into, in each dimension
    sampBin = [
        np.digitize(sample[:, i], edges[i])
        for i in xrange(Ndim)
    ]

    # Using `digitize`, values that fall on an edge are put in the right bin.
    # For the rightmost bin, we want values equal to the right
    # edge to be counted in the last bin, and not as an outlier.
    for i in xrange(Ndim):
        # Find the rounding precision
        decimal = int(-np.log10(dedges[i].min())) + 6
        # Find which points are on the rightmost edge.
        on_edge = np.where(np.around(sample[:, i], decimal) ==
                           np.around(edges[i][-1], decimal))[0]
        # Shift these points one bin to the left.
        sampBin[i][on_edge] -= 1

    # Compute the sample indices in the flattened statistic matrix.
    binnumbers = np.ravel_multi_index(sampBin, nbin)

    result = np.empty([Vdim, nbin.prod()], float)

    if statistic == 'mean':
        result.fill(np.nan)
        flatcount = np.bincount(binnumbers, None)
        a = flatcount.nonzero()
        for vv in xrange(Vdim):
            flatsum = np.bincount(binnumbers, values[vv])
            result[vv, a] = flatsum[a] / flatcount[a]
    elif statistic == 'std':
        result.fill(0)
        flatcount = np.bincount(binnumbers, None)
        a = flatcount.nonzero()
        for vv in xrange(Vdim):
            flatsum = np.bincount(binnumbers, values[vv])
            flatsum2 = np.bincount(binnumbers, values[vv] ** 2)
            result[vv, a] = np.sqrt(flatsum2[a] / flatcount[a] -
                                    (flatsum[a] / flatcount[a]) ** 2)
    elif statistic == 'count':
        result.fill(0)
        flatcount = np.bincount(binnumbers, None)
        a = np.arange(len(flatcount))
        result[:, a] = flatcount[np.newaxis, :]
    elif statistic == 'sum':
        result.fill(0)
        for vv in xrange(Vdim):
            flatsum = np.bincount(binnumbers, values[vv])
            a = np.arange(len(flatsum))
            result[vv, a] = flatsum
    elif statistic == 'median':
        result.fill(np.nan)
        for i in np.unique(binnumbers):
            for vv in xrange(Vdim):
                result[vv, i] = np.median(values[vv, binnumbers == i])
    elif statistic == 'min':
        result.fill(np.nan)
        for i in np.unique(binnumbers):
            for vv in xrange(Vdim):
                result[vv, i] = np.min(values[vv, binnumbers == i])
    elif statistic == 'max':
        result.fill(np.nan)
        for i in np.unique(binnumbers):
            for vv in xrange(Vdim):
                result[vv, i] = np.max(values[vv, binnumbers == i])
    elif callable(statistic):
        with np.errstate(invalid='ignore'), suppress_warnings() as sup:
            sup.filter(RuntimeWarning)
            try:
                null = statistic([])
            except:
                null = np.nan
        result.fill(null)
        for i in np.unique(binnumbers):
            for vv in xrange(Vdim):
                result[vv, i] = statistic(values[vv, binnumbers == i])

    # Shape into a proper matrix
    result = result.reshape(np.append(Vdim, nbin))

    # Remove outliers (indices 0 and -1 for each bin-dimension).
    core = [slice(None)] + Ndim * [slice(1, -1)]
    result = result[core]

    # Unravel binnumbers into an ndarray, each row the bins for each dimension
    if(expand_binnumbers and Ndim > 1):
        binnumbers = np.asarray(np.unravel_index(binnumbers, nbin))

    if np.any(result.shape[1:] != nbin - 2):
        raise RuntimeError('Internal Shape Error')

    # Reshape to have output (`reulst`) match input (`values`) shape
    result = result.reshape(input_shape[:-1] + list(nbin-2))

    return BinnedStatisticddResult(result, edges, binnumbers)


from collections import namedtuple

import numpy as np

from . import distributions

__all__ = ['_find_repeats', 'linregress', 'theilslopes']

def linregress(x, y=None):
    """
    Calculate a linear least-squares regression for two sets of measurements.

    Parameters
    ----------
    x, y : array_like
        Two sets of measurements.  Both arrays should have the same length.
        If only x is given (and y=None), then it must be a two-dimensional
        array where one dimension has length 2.  The two sets of measurements
        are then found by splitting the array along the length-2 dimension.

    Returns
    -------
    slope : float
        slope of the regression line
    intercept : float
        intercept of the regression line
    rvalue : float
        correlation coefficient
    pvalue : float
        two-sided p-value for a hypothesis test whose null hypothesis is
        that the slope is zero.
    stderr : float
        Standard error of the estimated gradient.

    See also
    --------
    optimize.curve_fit : Use non-linear least squares to fit a function to data.
    optimize.leastsq : Minimize the sum of squares of a set of equations.

    Examples
    --------
    >>> from scipy import stats
    >>> np.random.seed(12345678)
    >>> x = np.random.random(10)
    >>> y = np.random.random(10)
    >>> slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)

    # To get coefficient of determination (r_squared)

    >>> print("r-squared:", r_value**2)
    ('r-squared:', 0.080402268539028335)

    """
    TINY = 1.0e-20
    if y is None:  # x is a (2, N) or (N, 2) shaped array_like
        x = np.asarray(x)
        if x.shape[0] == 2:
            x, y = x
        elif x.shape[1] == 2:
            x, y = x.T
        else:
            msg = ("If only `x` is given as input, it has to be of shape "
                   "(2, N) or (N, 2), provided shape was %s" % str(x.shape))
            raise ValueError(msg)
    else:
        x = np.asarray(x)
        y = np.asarray(y)

    if x.size == 0 or y.size == 0:
        raise ValueError("Inputs must not be empty.")

    n = len(x)
    xmean = np.mean(x, None)
    ymean = np.mean(y, None)

    # average sum of squares:
    ssxm, ssxym, ssyxm, ssym = np.cov(x, y, bias=1).flat
    r_num = ssxym
    r_den = np.sqrt(ssxm * ssym)
    if r_den == 0.0:
        r = 0.0
    else:
        r = r_num / r_den
        # test for numerical error propagation
        if r > 1.0:
            r = 1.0
        elif r < -1.0:
            r = -1.0

    df = n - 2
    t = r * np.sqrt(df / ((1.0 - r + TINY)*(1.0 + r + TINY)))
    prob = 2 * distributions.t.sf(np.abs(t), df)
    slope = r_num / ssxm
    intercept = ymean - slope*xmean
    sterrest = np.sqrt((1 - r**2) * ssym / ssxm / df)

    LinregressResult = namedtuple('LinregressResult', ('slope', 'intercept',
                                                       'rvalue', 'pvalue',
                                                       'stderr'))
    return LinregressResult(slope, intercept, r, prob, sterrest)

def theilslopes(y, x=None, alpha=0.95):
    r"""
    Computes the Theil-Sen estimator for a set of points (x, y).

    `theilslopes` implements a method for robust linear regression.  It
    computes the slope as the median of all slopes between paired values.

    Parameters
    ----------
    y : array_like
        Dependent variable.
    x : array_like or None, optional
        Independent variable. If None, use ``arange(len(y))`` instead.
    alpha : float, optional
        Confidence degree between 0 and 1. Default is 95% confidence.
        Note that `alpha` is symmetric around 0.5, i.e. both 0.1 and 0.9 are
        interpreted as "find the 90% confidence interval".

    Returns
    -------
    medslope : float
        Theil slope.
    medintercept : float
        Intercept of the Theil line, as ``median(y) - medslope*median(x)``.
    lo_slope : float
        Lower bound of the confidence interval on `medslope`.
    up_slope : float
        Upper bound of the confidence interval on `medslope`.

    Notes
    -----
    The implementation of `theilslopes` follows [1]_. The intercept is
    not defined in [1]_, and here it is defined as ``median(y) -
    medslope*median(x)``, which is given in [3]_. Other definitions of
    the intercept exist in the literature. A confidence interval for
    the intercept is not given as this question is not addressed in
    [1]_.

    References
    ----------
    .. [1] P.K. Sen, "Estimates of the regression coefficient based on Kendall's tau",
           J. Am. Stat. Assoc., Vol. 63, pp. 1379-1389, 1968.
    .. [2] H. Theil, "A rank-invariant method of linear and polynomial
           regression analysis I, II and III",  Nederl. Akad. Wetensch., Proc.
           53:, pp. 386-392, pp. 521-525, pp. 1397-1412, 1950.
    .. [3] W.L. Conover, "Practical nonparametric statistics", 2nd ed.,
           John Wiley and Sons, New York, pp. 493.

    Examples
    --------
    >>> from scipy import stats
    >>> import matplotlib.pyplot as plt

    >>> x = np.linspace(-5, 5, num=150)
    >>> y = x + np.random.normal(size=x.size)
    >>> y[11:15] += 10  # add outliers
    >>> y[-5:] -= 7

    Compute the slope, intercept and 90% confidence interval.  For comparison,
    also compute the least-squares fit with `linregress`:

    >>> res = stats.theilslopes(y, x, 0.90)
    >>> lsq_res = stats.linregress(x, y)

    Plot the results. The Theil-Sen regression line is shown in red, with the
    dashed red lines illustrating the confidence interval of the slope (note
    that the dashed red lines are not the confidence interval of the regression
    as the confidence interval of the intercept is not included). The green
    line shows the least-squares fit for comparison.

    >>> fig = plt.figure()
    >>> ax = fig.add_subplot(111)
    >>> ax.plot(x, y, 'b.')
    >>> ax.plot(x, res[1] + res[0] * x, 'r-')
    >>> ax.plot(x, res[1] + res[2] * x, 'r--')
    >>> ax.plot(x, res[1] + res[3] * x, 'r--')
    >>> ax.plot(x, lsq_res[1] + lsq_res[0] * x, 'g-')
    >>> plt.show()

    """
    # We copy both x and y so we can use _find_repeats.
    y = np.array(y).flatten()
    if x is None:
        x = np.arange(len(y), dtype=float)
    else:
        x = np.array(x, dtype=float).flatten()
        if len(x) != len(y):
            raise ValueError("Incompatible lengths ! (%s<>%s)" % (len(y), len(x)))

    # Compute sorted slopes only when deltax > 0
    deltax = x[:, np.newaxis] - x
    deltay = y[:, np.newaxis] - y
    slopes = deltay[deltax > 0] / deltax[deltax > 0]
    slopes.sort()
    medslope = np.median(slopes)
    medinter = np.median(y) - medslope * np.median(x)
    # Now compute confidence intervals
    if alpha > 0.5:
        alpha = 1. - alpha

    z = distributions.norm.ppf(alpha / 2.)
    # This implements (2.6) from Sen (1968)
    _, nxreps = _find_repeats(x)
    _, nyreps = _find_repeats(y)
    nt = len(slopes)       # N in Sen (1968)
    ny = len(y)            # n in Sen (1968)
    # Equation 2.6 in Sen (1968):
    sigsq = 1/18. * (ny * (ny-1) * (2*ny+5) -
                     np.sum(k * (k-1) * (2*k + 5) for k in nxreps) -
                     np.sum(k * (k-1) * (2*k + 5) for k in nyreps))
    # Find the confidence interval indices in `slopes`
    sigma = np.sqrt(sigsq)
    Ru = min(int(np.round((nt - z*sigma)/2.)), len(slopes)-1)
    Rl = max(int(np.round((nt + z*sigma)/2.)) - 1, 0)
    delta = slopes[[Rl, Ru]]
    return medslope, medinter, delta[0], delta[1]

def _find_repeats(arr):
    # This function assumes it may clobber its input.
    if len(arr) == 0:
        return np.array(0, np.float64), np.array(0, np.intp)

    # XXX This cast was previously needed for the Fortran implementation,
    # should we ditch it?
    arr = np.asarray(arr, np.float64).ravel()
    arr.sort()

    # Taken from NumPy 1.9's np.unique.
    change = np.concatenate(([True], arr[1:] != arr[:-1]))
    unique = arr[change]
    change_idx = np.concatenate(np.nonzero(change) + ([arr.size],))
    freq = np.diff(change_idx)
    atleast2 = freq > 1
    return unique[atleast2], freq[atleast2]


import numpy as np
from numpy.testing import assert_equal, assert_approx_equal, \
                          assert_array_almost_equal
from nose.tools import assert_true

from sklearn.utils.testing import assert_less

from .. import make_classification
from .. import make_multilabel_classification
from .. import make_hastie_10_2
from .. import make_regression
from .. import make_blobs
from .. import make_friedman1
from .. import make_friedman2
from .. import make_friedman3
from .. import make_low_rank_matrix
from .. import make_sparse_coded_signal
from .. import make_sparse_uncorrelated
from .. import make_spd_matrix
from .. import make_swiss_roll
from .. import make_s_curve

def test_make_classification():
    X, y = make_classification(n_samples=100, n_features=20, n_informative=5,
                               n_redundant=1, n_repeated=1, n_classes=3,
                               n_clusters_per_class=1, hypercube=False,
                               shift=None, scale=None, weights=[0.1, 0.25],
                               random_state=0)

    assert_equal(X.shape, (100, 20), "X shape mismatch")
    assert_equal(y.shape, (100,), "y shape mismatch")
    assert_equal(np.unique(y).shape, (3,), "Unexpected number of classes")
    assert_equal(sum(y == 0), 10, "Unexpected number of samples in class #0")
    assert_equal(sum(y == 1), 25, "Unexpected number of samples in class #1")
    assert_equal(sum(y == 2), 65, "Unexpected number of samples in class #2")

def test_make_multilabel_classification():
    for allow_unlabeled, min_length in zip((True, False), (0, 1)):
        X, Y = make_multilabel_classification(n_samples=100, n_features=20,
                                              n_classes=3, random_state=0,
                                              allow_unlabeled=allow_unlabeled)
        assert_equal(X.shape, (100, 20), "X shape mismatch")
        if not allow_unlabeled:
            assert_equal(max([max(y) for y in Y]), 2)
        assert_equal(min([len(y) for y in Y]), min_length)
        assert_true(max([len(y) for y in Y]) <= 3)

def test_make_hastie_10_2():
    X, y = make_hastie_10_2(n_samples=100, random_state=0)
    assert_equal(X.shape, (100, 10), "X shape mismatch")
    assert_equal(y.shape, (100,), "y shape mismatch")
    assert_equal(np.unique(y).shape, (2,), "Unexpected number of classes")

def test_make_regression():
    X, y, c = make_regression(n_samples=100, n_features=10, n_informative=3,
                              effective_rank=5, coef=True, bias=0.0,
                              noise=1.0, random_state=0)

    assert_equal(X.shape, (100, 10), "X shape mismatch")
    assert_equal(y.shape, (100,), "y shape mismatch")
    assert_equal(c.shape, (10,), "coef shape mismatch")
    assert_equal(sum(c != 0.0), 3, "Unexpected number of informative features")

    # Test that y ~= np.dot(X, c) + bias + N(0, 1.0)
    assert_approx_equal(np.std(y - np.dot(X, c)), 1.0, significant=2)

def test_make_regression_multitarget():
    X, y, c = make_regression(n_samples=100, n_features=10, n_informative=3,
                              n_targets=3, coef=True, noise=1., random_state=0)

    assert_equal(X.shape, (100, 10), "X shape mismatch")
    assert_equal(y.shape, (100, 3), "y shape mismatch")
    assert_equal(c.shape, (10, 3), "coef shape mismatch")
    assert_equal(sum(c != 0.0), 3, "Unexpected number of informative features")

    # Test that y ~= np.dot(X, c) + bias + N(0, 1.0)
    assert_approx_equal(np.std(y - np.dot(X, c)), 1.0, significant=2)

def test_make_blobs():
    X, y = make_blobs(n_samples=50, n_features=2,
                      centers=[[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]],
                      random_state=0)

    assert_equal(X.shape, (50, 2), "X shape mismatch")
    assert_equal(y.shape, (50,), "y shape mismatch")
    assert_equal(np.unique(y).shape, (3,), "Unexpected number of blobs")

def test_make_friedman1():
    X, y = make_friedman1(n_samples=5, n_features=10, noise=0.0,
                          random_state=0)

    assert_equal(X.shape, (5, 10), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")

    assert_array_almost_equal(y, 10 * np.sin(np.pi * X[:, 0] * X[:, 1])
                                 + 20 * (X[:, 2] - 0.5) ** 2 \
                                 + 10 * X[:, 3] + 5 * X[:, 4])

def test_make_friedman2():
    X, y = make_friedman2(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 4), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")

    assert_array_almost_equal(y, (X[:, 0] ** 2
                                 + (X[:, 1] * X[:, 2]
                                    - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5)

def test_make_friedman3():
    X, y = make_friedman3(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 4), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")

    assert_array_almost_equal(y, np.arctan((X[:, 1] * X[:, 2]
                                            - 1 / (X[:, 1] * X[:, 3]))
                                           / X[:, 0]))

def test_make_low_rank_matrix():
    X = make_low_rank_matrix(n_samples=50, n_features=25, effective_rank=5,
                             tail_strength=0.01, random_state=0)

    assert_equal(X.shape, (50, 25), "X shape mismatch")

    from numpy.linalg import svd
    u, s, v = svd(X)
    assert_less(sum(s) - 5, 0.1, "X rank is not approximately 5")

def test_make_sparse_coded_signal():
    Y, D, X = make_sparse_coded_signal(n_samples=5, n_components=8,
                                           n_features=10, n_nonzero_coefs=3,
                                           random_state=0)
    assert_equal(Y.shape, (10, 5), "Y shape mismatch")
    assert_equal(D.shape, (10, 8), "D shape mismatch")
    assert_equal(X.shape, (8, 5), "X shape mismatch")
    for col in X.T:
        assert_equal(len(np.flatnonzero(col)), 3, 'Non-zero coefs mismatch')
    assert_equal(np.dot(D, X), Y)
    assert_array_almost_equal(np.sqrt((D ** 2).sum(axis=0)),
                              np.ones(D.shape[1]))

def test_make_sparse_uncorrelated():
    X, y = make_sparse_uncorrelated(n_samples=5, n_features=10, random_state=0)

    assert_equal(X.shape, (5, 10), "X shape mismatch")
    assert_equal(y.shape, (5,), "y shape mismatch")

def test_make_spd_matrix():
    X = make_spd_matrix(n_dim=5, random_state=0)

    assert_equal(X.shape, (5, 5), "X shape mismatch")
    assert_array_almost_equal(X, X.T)

    from numpy.linalg import eig
    eigenvalues, _ = eig(X)
    assert_equal(eigenvalues > 0, np.array([True] * 5),
                 "X is not positive-definite")

def test_make_swiss_roll():
    X, t = make_swiss_roll(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 3), "X shape mismatch")
    assert_equal(t.shape, (5,), "t shape mismatch")
    assert_equal(X[:, 0], t * np.cos(t))
    assert_equal(X[:, 2], t * np.sin(t))

def test_make_s_curve():
    X, t = make_s_curve(n_samples=5, noise=0.0, random_state=0)

    assert_equal(X.shape, (5, 3), "X shape mismatch")
    assert_equal(t.shape, (5,), "t shape mismatch")
    assert_equal(X[:, 0], np.sin(t))
    assert_equal(X[:, 2], np.sign(t) * (np.cos(t) - 1))


from __future__ import division

import numpy as np
import scipy.sparse as sp
from scipy.misc import comb as combinations
from numpy.testing import assert_array_almost_equal
from sklearn.utils.random import sample_without_replacement
from sklearn.utils.random import random_choice_csc

from sklearn.utils.testing import (
    assert_raises,
    assert_equal,
    assert_true)

###############################################################################
# test custom sampling without replacement algorithm
###############################################################################
def test_invalid_sample_without_replacement_algorithm():
    assert_raises(ValueError, sample_without_replacement, 5, 4, "unknown")

def test_sample_without_replacement_algorithms():
    methods = ("auto", "tracking_selection", "reservoir_sampling", "pool")

    for m in methods:
        def sample_without_replacement_method(n_population, n_samples,
                                              random_state=None):
            return sample_without_replacement(n_population, n_samples,
                                              method=m,
                                              random_state=random_state)

        check_edge_case_of_sample_int(sample_without_replacement_method)
        check_sample_int(sample_without_replacement_method)
        check_sample_int_distribution(sample_without_replacement_method)

def check_edge_case_of_sample_int(sample_without_replacement):

    # n_population < n_sample
    assert_raises(ValueError, sample_without_replacement, 0, 1)
    assert_raises(ValueError, sample_without_replacement, 1, 2)

    # n_population == n_samples
    assert_equal(sample_without_replacement(0, 0).shape, (0, ))

    assert_equal(sample_without_replacement(1, 1).shape, (1, ))

    # n_population >= n_samples
    assert_equal(sample_without_replacement(5, 0).shape, (0, ))
    assert_equal(sample_without_replacement(5, 1).shape, (1, ))

    # n_population < 0 or n_samples < 0
    assert_raises(ValueError, sample_without_replacement, -1, 5)
    assert_raises(ValueError, sample_without_replacement, 5, -1)

def check_sample_int(sample_without_replacement):
    # This test is heavily inspired from test_random.py of python-core.
    #
    # For the entire allowable range of 0 <= k <= N, validate that
    # the sample is of the correct length and contains only unique items
    n_population = 100

    for n_samples in range(n_population + 1):
        s = sample_without_replacement(n_population, n_samples)
        assert_equal(len(s), n_samples)
        unique = np.unique(s)
        assert_equal(np.size(unique), n_samples)
        assert_true(np.all(unique < n_population))

    # test edge case n_population == n_samples == 0
    assert_equal(np.size(sample_without_replacement(0, 0)), 0)

def check_sample_int_distribution(sample_without_replacement):
    # This test is heavily inspired from test_random.py of python-core.
    #
    # For the entire allowable range of 0 <= k <= N, validate that
    # sample generates all possible permutations
    n_population = 10

    # a large number of trials prevents false negatives without slowing normal
    # case
    n_trials = 10000

    for n_samples in range(n_population):
        # Counting the number of combinations is not as good as counting the
        # the number of permutations. However, it works with sampling algorithm
        # that does not provide a random permutation of the subset of integer.
        n_expected = combinations(n_population, n_samples, exact=True)

        output = {}
        for i in range(n_trials):
            output[frozenset(sample_without_replacement(n_population,
                                                        n_samples))] = None

            if len(output) == n_expected:
                break
        else:
            raise AssertionError(
                "number of combinations != number of expected (%s != %s)" %
                (len(output), n_expected))

def test_random_choice_csc(n_samples=10000, random_state=24):
    # Explicit class probabilities
    classes = [np.array([0, 1]),  np.array([0, 1, 2])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]

    got = random_choice_csc(n_samples, classes, class_probabilites,
                            random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel()) / float(n_samples)
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)

    # Implicit class probabilities
    classes = [[0, 1],  [1, 2]]  # test for array-like support
    class_probabilites = [np.array([0.5, 0.5]), np.array([0, 1/2, 1/2])]

    got = random_choice_csc(n_samples=n_samples,
                            classes=classes,
                            random_state=random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel()) / float(n_samples)
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)

    # Edge case probabilities 1.0 and 0.0
    classes = [np.array([0, 1]),  np.array([0, 1, 2])]
    class_probabilites = [np.array([1.0, 0.0]), np.array([0.0, 1.0, 0.0])]

    got = random_choice_csc(n_samples, classes, class_probabilites,
                            random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel(),
                        minlength=len(class_probabilites[k])) / n_samples
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)

    # One class target data
    classes = [[1],  [0]]  # test for array-like support
    class_probabilites = [np.array([0.0, 1.0]), np.array([1.0])]

    got = random_choice_csc(n_samples=n_samples,
                            classes=classes,
                            random_state=random_state)
    assert_true(sp.issparse(got))

    for k in range(len(classes)):
        p = np.bincount(got.getcol(k).toarray().ravel()) / n_samples
        assert_array_almost_equal(class_probabilites[k], p, decimal=1)

def test_random_choice_csc_errors():
    # the length of an array in classes and class_probabilites is mismatched
    classes = [np.array([0, 1]),  np.array([0, 1, 2, 3])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)

    # the class dtype is not supported
    classes = [np.array(["a", "1"]),  np.array(["z", "1", "2"])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)

    # the class dtype is not supported
    classes = [np.array([4.2, 0.1]),  np.array([0.1, 0.2, 9.4])]
    class_probabilites = [np.array([0.5, 0.5]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)

    # Given probabilities don't sum to 1
    classes = [np.array([0, 1]),  np.array([0, 1, 2])]
    class_probabilites = [np.array([0.5, 0.6]), np.array([0.6, 0.1, 0.3])]
    assert_raises(ValueError, random_choice_csc, 4, classes,
                  class_probabilites, 1)


"""
Agglomerative clustering with different metrics
===============================================

Demonstrates the effect of different metrics on the hierarchical clustering.

The example is engineered to show the effect of the choice of different
metrics. It is applied to waveforms, which can be seen as
high-dimensional vector. Indeed, the difference between metrics is
usually more pronounced in high dimension (in particular for euclidean
and cityblock).

We generate data from three groups of waveforms. Two of the waveforms
(waveform 1 and waveform 2) are proportional one to the other. The cosine
distance is invariant to a scaling of the data, as a result, it cannot
distinguish these two waveforms. Thus even with no noise, clustering
using this distance will not separate out waveform 1 and 2.

We add observation noise to these waveforms. We generate very sparse
noise: only 6% of the time points contain noise. As a result, the
l1 norm of this noise (ie "cityblock" distance) is much smaller than it's
l2 norm ("euclidean" distance). This can be seen on the inter-class
distance matrices: the values on the diagonal, that characterize the
spread of the class, are much bigger for the Euclidean distance than for
the cityblock distance.

When we apply clustering to the data, we find that the clustering
reflects what was in the distance matrices. Indeed, for the Euclidean
distance, the classes are ill-separated because of the noise, and thus
the clustering does not separate the waveforms. For the cityblock
distance, the separation is good and the waveform classes are recovered.
Finally, the cosine distance does not separate at all waveform 1 and 2,
thus the clustering puts them in the same cluster.
"""
# Author: Gael Varoquaux
# License: BSD 3-Clause or CC-0

import matplotlib.pyplot as plt
import numpy as np

from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics import pairwise_distances

np.random.seed(0)

# Generate waveform data
n_features = 2000
t = np.pi * np.linspace(0, 1, n_features)

def sqr(x):
    return np.sign(np.cos(x))

X = list()
y = list()
for i, (phi, a) in enumerate([(.5, .15), (.5, .6), (.3, .2)]):
    for _ in range(30):
        phase_noise = .01 * np.random.normal()
        amplitude_noise = .04 * np.random.normal()
        additional_noise = 1 - 2 * np.random.rand(n_features)
        # Make the noise sparse
        additional_noise[np.abs(additional_noise) < .997] = 0

        X.append(12 * ((a + amplitude_noise)
                 * (sqr(6 * (t + phi + phase_noise)))
                 + additional_noise))
        y.append(i)

X = np.array(X)
y = np.array(y)

n_clusters = 3

labels = ('Waveform 1', 'Waveform 2', 'Waveform 3')

# Plot the ground-truth labelling
plt.figure()
plt.axes([0, 0, 1, 1])
for l, c, n in zip(range(n_clusters), 'rgb',
                   labels):
    lines = plt.plot(X[y == l].T, c=c, alpha=.5)
    lines[0].set_label(n)

plt.legend(loc='best')

plt.axis('tight')
plt.axis('off')
plt.suptitle("Ground truth", size=20)

# Plot the distances
for index, metric in enumerate(["cosine", "euclidean", "cityblock"]):
    avg_dist = np.zeros((n_clusters, n_clusters))
    plt.figure(figsize=(5, 4.5))
    for i in range(n_clusters):
        for j in range(n_clusters):
            avg_dist[i, j] = pairwise_distances(X[y == i], X[y == j],
                                                metric=metric).mean()
    avg_dist /= avg_dist.max()
    for i in range(n_clusters):
        for j in range(n_clusters):
            plt.text(i, j, '%5.3f' % avg_dist[i, j],
                     verticalalignment='center',
                     horizontalalignment='center')

    plt.imshow(avg_dist, interpolation='nearest', cmap=plt.cm.gnuplot2,
               vmin=0)
    plt.xticks(range(n_clusters), labels, rotation=45)
    plt.yticks(range(n_clusters), labels)
    plt.colorbar()
    plt.suptitle("Interclass %s distances" % metric, size=18)
    plt.tight_layout()

# Plot clustering results
for index, metric in enumerate(["cosine", "euclidean", "cityblock"]):
    model = AgglomerativeClustering(n_clusters=n_clusters,
                                    linkage="average", affinity=metric)
    model.fit(X)
    plt.figure()
    plt.axes([0, 0, 1, 1])
    for l, c in zip(np.arange(model.n_clusters), 'rgbk'):
        plt.plot(X[model.labels_ == l].T, c=c, alpha=.5)
    plt.axis('tight')
    plt.axis('off')
    plt.suptitle("AgglomerativeClustering(affinity=%s)" % metric, size=20)

plt.show()


#Filename:  hTestLimit.py
#Author:    Matt Strader
#
#This script opens a list of observed photon phases,
import numpy as np
import tables
import numexpr
import matplotlib.pyplot as plt
import multiprocessing
import functools
import time

from kuiper.kuiper import kuiper,kuiper_FPP
from kuiper.htest import h_test,h_fpp,h_test2
from pulsarUtils import nSigma,plotPulseProfile
from histMetrics import kuiperFpp,hTestFpp
from inverseTransformSampling import inverseTransformSampler

def hTestTrial(iTrial,nPhotons,photonPulseFraction,pulseModel,pulseModelQueryPoints):
    np.random.seed(int((time.time()+iTrial)*1e6))
    modelSampler = inverseTransformSampler(pdf=pulseModel,queryPoints=pulseModelQueryPoints)

    nPulsePhotons = int(np.floor(photonPulseFraction*nPhotons))
    nBackgroundPhotons = int(np.ceil((1.-photonPulseFraction) * nPhotons))

    simPulsePhotons = modelSampler(nPulsePhotons)
    #background photons come from a uniform distribution
    simBackgroundPhotons = np.random.random(nBackgroundPhotons)
    simPhases = np.append(simPulsePhotons,simBackgroundPhotons)

    simHDict = h_test2(simPhases)
    simH,simM,simPval,simFourierCoeffs = simHDict['H'],simHDict['M'],simHDict['fpp'],simHDict['cs']

    print '{} - H,M,fpp,sig:'.format(iTrial),simH,simM,simPval
    return {'H':simH,'M':simM,'fpp':simPval}

if __name__=='__main__':

    path = '/Scratch/dataProcessing/J0337/masterPhotons3.h5'

    wvlStart = 4000.
    wvlEnd = 5500.
    bLoadFromPl = True
    nPhaseBins = 20
    hTestPath = '/Scratch/dataProcessing/J0337/hTestResults_withProfiles_{}-{}.npz'.format(wvlStart,wvlEnd)
    phaseBinEdges = np.linspace(0.,1.,nPhaseBins+1)

    if bLoadFromPl:
        photFile = tables.openFile(path,'r')
        photTable = photFile.root.photons.photTable
        phases = photTable.readWhere('(wvlStart < wavelength) & (wavelength < wvlEnd)')['phase']
        photFile.close()
        print 'cut wavelengths to range ({},{})'.format(wvlStart,wvlEnd)

        nPhotons = len(phases)
        print nPhotons,'real photons read'

        observedProfile,_ = np.histogram(phases,bins=phaseBinEdges)
        observedProfile = 1.0*observedProfile
        observedProfileErrors = np.sqrt(observedProfile)

        #Do H-test
        hDict = h_test2(phases)
        H,M,pval,fourierCoeffs = hDict['H'],hDict['M'],hDict['fpp'],hDict['cs']

        print 'h-test on real data'
        print 'H,M,fpp:',H,M,pval
        print nSigma(1-pval),'sigmas'

        #h_test2 calculates all fourierCoeffs out to 20, but for the fourier model, we only want the ones out to order M, which optimizes the Zm^2 metric
        truncatedFourierCoeffs = fourierCoeffs[0:M]
        print 'fourier coeffs:',truncatedFourierCoeffs

        #for the model, we want the negative modes as well as positve, so add them
        modelFourierCoeffs = np.concatenate([truncatedFourierCoeffs[::-1],[1.],np.conj(truncatedFourierCoeffs)])
        #make array of mode numbers
        modes = np.arange(-len(truncatedFourierCoeffs),len(truncatedFourierCoeffs)+1)

        #save so next time we can set bLoadFromPl=False
        np.savez(hTestPath,H=H,M=M,pval=pval,fourierCoeffs=fourierCoeffs,nPhotons=nPhotons,wvlRange=(wvlStart,wvlEnd),modelFourierCoeffs=modelFourierCoeffs,modes=modes,observedProfile=observedProfile,observedProfileErrors=observedProfileErrors,phaseBinEdges=phaseBinEdges)

    else:
        #Load values from previous run, when we had bLoadFromPl=True
        hTestDict = np.load(hTestPath)
        H,M,pval,fourierCoeffs,nPhotons,modelFourierCoeffs,modes = hTestDict['H'],hTestDict['M'],hTestDict['pval'],hTestDict['fourierCoeffs'],hTestDict['nPhotons'],hTestDict['modelFourierCoeffs'],hTestDict['modes']
        observedProfile,observedProfileErrors,phaseBinEdges = hTestDict['observedProfile'],hTestDict['observedProfileErrors'],hTestDict['phaseBinEdges']

        print 'h-test on real data'
        print 'H,M,fpp:',H,M,pval
        print nSigma(1-pval),'sigmas'

    #Plot the observed profile
    fig,ax = plt.subplots(1,1)
    plotPulseProfile(phaseBinEdges,observedProfile,profileErrors=observedProfileErrors,color='k',plotDoublePulse=False,label='observed',ax=ax)
    ax.set_ylabel('counts')
    ax.set_xlabel('phase')
    ax.set_title('Observed Folded Light Curve {}-{} nm'.format(wvlStart/10.,wvlEnd/10.))

    #make as set of x points for the pulse model we'll make
    #Do NOT include x=0, or the inverted function will have a jump that causes an excess of samples
    #at phase=0
    nSmoothPlotPoints=1000
    pulseModelQueryPoints = np.linspace(1./nSmoothPlotPoints,1,nSmoothPlotPoints)

    def modelProfile(thetas):
        return np.sum( modelFourierCoeffs * np.exp(2.j*np.pi*modes*thetas[:,np.newaxis]),axis=1)
    lightCurveModel = np.abs(modelProfile(pulseModelQueryPoints))
    #for this test we only want the model to be the pulsed component.  We will add a DC offset later
    pulseModel = lightCurveModel - np.min(lightCurveModel)

    #initialPhotonPulseFraction = 1.*np.sum(pulseModel) / np.sum(lightCurveModel)
    photonPulseFraction=15400./nPhotons #skip to previously determined answer
    print 'photon fraction',photonPulseFraction

    #get samples with distribution of the modelProfile
    #modelSampler = inverseTransformSampler(pdf=lightCurveModel,queryPoints=pulseModelQueryPoints)
    modelSampler = inverseTransformSampler(pdf=pulseModel,queryPoints=pulseModelQueryPoints)

    nTrials = 1
    #for each trial run the h test on a set of photon phases with our model profile, and with the pulse fraction specified
    #we want to make a distribution of H values for this pulse fraction, model, and number of photons

    #make a function that only takes the trial number (as an identifier)
    mappableHTestTrial = functools.partial(hTestTrial,pulseModel=pulseModel,
            pulseModelQueryPoints=pulseModelQueryPoints,nPhotons=nPhotons,
            photonPulseFraction=photonPulseFraction)
    pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()-3)#leave a few processors for other people
    outDicts = pool.map(mappableHTestTrial,np.arange(nTrials))

    simHs = np.array([out['H'] for out in outDicts])
    simPvals = np.array([out['fpp'] for out in outDicts])
    #save the resulting list of H vals
    np.savez('sim3-h-{}.npz'.format(nTrials),simHs=simHs,simPvals=simPvals,pval=pval,H=H,photonPulseFraction=photonPulseFraction,nPhotons=nPhotons)

    #make a model profile once more for a plot
    modelSampler = inverseTransformSampler(pdf=pulseModel,queryPoints=pulseModelQueryPoints)

    nPulsePhotons = int(np.floor(photonPulseFraction*nPhotons))
    nBackgroundPhotons = int(np.ceil((1.-photonPulseFraction) * nPhotons))

    simPulsePhotons = modelSampler(nPulsePhotons)
    #background photons come from a uniform distribution
    simBackgroundPhotons = np.random.random(nBackgroundPhotons)
    #put them together for the full profile
    simPhases = np.append(simPulsePhotons,simBackgroundPhotons)

    #make a binned phase profile to plot
    simProfile,_ = np.histogram(simPhases,bins=phaseBinEdges)
    simProfileErrors = np.sqrt(simProfile)#assume Poisson errors
    meanLevel = np.mean(simProfile)

    fig,ax = plt.subplots(1,1)
    ax.plot(pulseModelQueryPoints,meanLevel*lightCurveModel,color='r')
    plotPulseProfile(phaseBinEdges,simProfile,profileErrors=simProfileErrors,color='b',plotDoublePulse=False,label='sim',ax=ax)
    ax.set_title('Simulated profile')
    #
    #plt.show()

    print '{} trials'.format(len(simHs))
    print 'observed fpp:',pval

    frac = 1.*np.sum(simPvals<pval)/len(simPvals)
    print 'fraction of trials with H below observed fpp:',frac
    #hHist,hBinEdges = np.histogram(simHs,bins=100,density=True)
    fppHist,fppBinEdges = np.histogram(simPvals,bins=100,density=True)

    if nTrials > 1:
        fig,ax = plt.subplots(1,1)
        ax.plot(fppBinEdges[0:-1],fppHist,drawstyle='steps-post',color='k')
        ax.axvline(pval,color='r')
        ax.set_xlabel('fpp')
        ax.set_ylabel('frequency')

        ax.set_title('Distribution of H for model profile')

    magG = 17.93
    sineMagDiff = -2.5*np.log10(photonPulseFraction)
    print 'SDSS magnitude g: {:.2f}'.format(magG)
    print 'magnitude difference: {:.2f}'.format(sineMagDiff)
    print 'limiting g mag: {:.2f}'.format(magG+sineMagDiff)

    plt.show()


#!/usr/bin/env python
# (c) Stefan Countryman, 2016-2017

DESC="""Plot an IRIG-B signal read from stdin. Assumes that the timeseries
is a sequence of newline-delimited float literals."""
FAST_CHANNEL_BITRATE = 16384  # for IRIG-B, DuoTone, etc.

# THE REST OF THE IMPORTS ARE AFTER THIS IF STATEMENT.
# Quits immediately on --help or -h flags to skip slow imports when you just
# want to read the help documentation.
if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description=DESC)
    # TODO: make this -i and --ifo instead of detector.
    parser.add_argument("--detector",
                        help=("the detector; used in the title of the output "
                              "plot"))
    parser.add_argument("-O", "--outfile",
                        help="the filename of the generated plot")
    parser.add_argument("-T", "--timeseries",
                        help="copy from stdin to stdout while reading",
                        action="store_true")
    parser.add_argument("-A", "--actualtime",
                        help=("actual time signal was recorded "
                              "(appears in title)"))
    args = parser.parse_args()

# Force matplotlib to not use any Xwindows backend. NECESSARY ON CLUSTER.
import matplotlib
matplotlib.use('Agg')
import sys
import time
import numpy as np
import matplotlib.pyplot as plt
import geco_irig_decode

def read_timeseries_stdin(num_lines, cat_to_stdout=False):
    """Read in newline-delimited numerical data from stdin; don't read more
    than a second worth of data. If cat_to_stdout is True, print data that
    has been read in back to stdout (useful for piped commands)."""
    timeseries = np.zeros(num_lines)
    line = ""
    i = 0
    while i < num_lines:
        line = float(sys.stdin.readline())
        timeseries[i] = line
        if cat_to_stdout:
            print(line)
        i += 1
    return timeseries

def irigb_decoded_title(timeseries, IFO=None, actual_time=None):
    """Get a title for an IRIG-B timeseries plot that includes the decoded
    time in the timeseries itself."""
    # get the detector name
    if IFO is None:
        detector_suffix = ""
    else:
        detector_suffix = " at " + IFO

    # get the actual time of recording, if provided
    if actual_time is None:
        actual_time_str = ""
    else:
        actual_time_str = "\nActual Time: {}".format(actual_time)

    # add title and so on
    try:
        decoded_time = geco_irig_decode.get_date_from_timeseries(timeseries)
        decoded_time_str = decoded_time.strftime('%a %b %d %X %Y')
    except ValueError as e:
        decoded_time_str = "COULD NOT DECODE TIME"
    fmt = "One Second of IRIG-B Signal{}\nDecoded Time: {}{}"
    return fmt.format(detector_suffix, decoded_time_str, actual_time_str)

def irigb_output_filename(outfile=None):
    """Get the output filename for an IRIG-B plot."""
    if outfile is None:
        output_filename = "irigb-plot-made-at-" + str(time.time()) + ".png"
    else:
        output_filename = outfile
        # append .png if not already there
        if output_filename.split(".")[-1] != "png":
            output_filename += ".png"
    return output_filename

def plot_with_zoomed_views(timeseries, title, num_subdivs=5, dt=1.,
                           output_filename=None, overlay=False, linewidth=1):
    """Plot a timeseries and produce num_subdivs subplots that show equal-sized
    subdivisions of the full timeseries data to show details (good for
    high-bitrate timeseries). If you want to keep plotting data to the same
    figure, set 'overlay=True', and the current figure will be plotted to."""
    bitrate = int(len(timeseries) / float(dt))
    times = np.linspace(0, 1, num=bitrate, endpoint=False)

    # find max and min values in timeseries; use these to set plot boundaries
    yrange = timeseries.max() - timeseries.min()
    ymax = timeseries.max() + 0.1*yrange
    ymin = timeseries.min() - 0.1*yrange

    if not overlay:
        plt.figure()
    # print("making plot")
    plt.gcf().set_figwidth(7)
    plt.gcf().set_figheight(4+1.2*num_subdivs) # ~1.2in height per zoomed plot
    # plot the full second on the first row; lines should be black ('k' option).
    plt.subplot(num_subdivs + 1, 1, 1)
    plt.ylim(ymin, ymax)
    plt.plot(times, timeseries, 'k', linewidth=linewidth)
    plt.tick_params(axis='y', labelsize='small')
    # make num_subdivs subplots to better show the full second
    for i in range(num_subdivs):
        # print("making plot " + str(i))
        plt.subplot(num_subdivs+1, 1, i+2)
        plt.ylim(ymin, ymax)
        plt.xlim(float(i)/num_subdivs, (float(i)+1)/num_subdivs)
        start = bitrate*i     // num_subdivs
        end   = bitrate*(i+1) // num_subdivs
        plt.plot(times[start:end], timeseries[start:end], 'k',
		 linewidth=linewidth)
        plt.tick_params(axis='y', labelsize='small')
    plt.suptitle(title)
    plt.xlabel("Time since start of second [$s$]")
    # print("saving plot")
    plt.subplots_adjust(left=0.125, right=0.9, bottom=0.1, top=0.9, wspace=0.2,
                        hspace=0.5)
    if not (output_filename is None):
        plt.savefig(output_filename)

    return plt

if __name__ == '__main__':
    timeseries = read_timeseries_stdin(FAST_CHANNEL_BITRATE,
                                       cat_to_stdout=args.timeseries)
    title = irigb_decoded_title(timeseries, args.detector, args.actualtime)
    output_filename = irigb_output_filename(args.outfile)
    plot_with_zoomed_views(timeseries, title, num_subdivs=5, dt=1.,
                           output_filename=output_filename)


"""
==================================================
Plot different SVM classifiers in the iris dataset
==================================================

Comparison of different linear SVM classifiers on a 2D projection of the iris
dataset. We only consider the first 2 features of this dataset:

- Sepal length
- Sepal width

This example shows how to plot the decision surface for four SVM classifiers
with different kernels.

The linear models ``LinearSVC()`` and ``SVC(kernel='linear')`` yield slightly
different decision boundaries. This can be a consequence of the following
differences:

- ``LinearSVC`` minimizes the squared hinge loss while ``SVC`` minimizes the
  regular hinge loss.

- ``LinearSVC`` uses the One-vs-All (also known as One-vs-Rest) multiclass
  reduction while ``SVC`` uses the One-vs-One multiclass reduction.

Both linear models have linear decision boundaries (intersecting hyperplanes)
while the non-linear kernel models (polynomial or Gaussian RBF) have more
flexible non-linear decision boundaries with shapes that depend on the kind of
kernel and its parameters.

.. NOTE:: while plotting the decision function of classifiers for toy 2D
   datasets can help get an intuitive understanding of their respective
   expressive power, be aware that those intuitions don't always generalize to
   more realistic high-dimensional problems.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target

h = .02  # step size in the mesh

# we create an instance of SVM and fit out data. We do not scale our
# data since we want to plot the support vectors
C = 1.0  # SVM regularization parameter
svc = svm.SVC(kernel='linear', C=C).fit(X, y)
rbf_svc = svm.SVC(kernel='rbf', gamma=0.7, C=C).fit(X, y)
poly_svc = svm.SVC(kernel='poly', degree=3, C=C).fit(X, y)
lin_svc = svm.LinearSVC(C=C).fit(X, y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

# title for the plots
titles = ['SVC with linear kernel',
          'LinearSVC (linear kernel)',
          'SVC with RBF kernel',
          'SVC with polynomial (degree 3) kernel']

for i, clf in enumerate((svc, lin_svc, rbf_svc, poly_svc)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, m_max]x[y_min, y_max].
    plt.subplot(2, 2, i + 1)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.8)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)
    plt.xlabel('Sepal length')
    plt.ylabel('Sepal width')
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.xticks(())
    plt.yticks(())
    plt.title(titles[i])

plt.show()


#!/usr/bin/env python

import sys
import os
import string

import numpy as np
import matplotlib.pyplot as plt
import re

def main():
	dir = sys.argv[1]

	if len(sys.argv) == 1:
		dict = create_dict(dir)
		draw_graph(dict)
	else:
		for i in range(2, len(sys.argv)):
			dict = create_dict(dir, sys.argv[i])
			draw_graph(dict, sys.argv[i])

def create_dict(rootdir, match='.*'):
	pattern = re.compile(match)
	dict = {}

	for branch in os.listdir(rootdir):
		branch_dict = {}
		for test in os.listdir(os.path.join(rootdir, branch)):
			if pattern.match(test):
				file = open(os.path.join(rootdir, branch, test))
				str = file.readline()
				str = str.strip()
				start = str.find("=")
				if start != -1:
					branch_dict[test] = round(string.atof(str[start+1:]),2)
				else:
					branch_dict[test] = -1.
		dict[branch] = branch_dict

	return dict

def get_all_test_name(dict):
	for branch in dict:
		return dict[branch].keys()

def get_branches(dict):
	return dict.keys()

def compare_by_branch(dict):
	def local_print(test, d):
		print test
		for t in d:
			print "\t" +  t + "\t" + str(d[t])
		print

	for test in get_all_test_name(dict):
		local_dict = {}
		for branch in dict:
			local_dict[branch] = dict[branch][test]

		local_print(test, local_dict)

### Unused ###

def short_test_name(long_name):
	return long_name[long_name.rfind('.Test')+5:]

def draw_graph(dict, title):

	def autolabel(rects):
		for rect in rects:
			height = rect.get_height()
			ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%d'%int(height),
			        ha='center', va='bottom')

	def set_legend(bars, branches):
		bs = ()
		for bar in bars:
			bs = bs + (bar[0],)
		ax.legend( bs, branches)

	colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']
	branches = get_branches(dict)
	all_tests = get_all_test_name(dict)

	N = len(all_tests)
	ind = np.arange(N)
	width = 0.35

	fig = plt.figure()
	ax = fig.add_subplot(111)

	data_sets = []
	for branch in branches:
		data =()
		for test in all_tests:
			data = data + (dict[branch].get(test, 0),)
		data_sets.append(data)

	bars = []
	counter = 0
	for data in data_sets:
		bar = ax.bar(ind + (counter*width), data, width, color=colors[counter])
		bars.append(bar)
		counter += 1

	# add some
	ax.set_ylabel('Perf')
	ax.set_title('Branch perf comparison for ' + title)
	ax.set_xticks(ind+width)
	ax.set_xticklabels(map(short_test_name, all_tests))

	set_legend(bars, branches)

	for bar in bars:
		autolabel(bar)

	plt.savefig(title + ".png")

if __name__ == "__main__":
    main()


'''

    A class for running a single instance of a NeuroML model by generating a
    LEMS file and using pyNeuroML to run in a chosen simulator

'''

import sys
import time

from pyneuroml import pynml

from pyneuroml.lems import generate_lems_file_for_neuroml

try:
    import pyelectro  #  Not used here, just for checking installation
except:
    print('>> Note: pyelectro from https://github.com/pgleeson/pyelectro is required!')
    exit()

try:
    import neurotune  #  Not used here, just for checking installation
except:
    print('>> Note: neurotune from https://github.com/pgleeson/neurotune is required!')
    exit()

class NeuroMLSimulation(object):

    def __init__(self,
                 reference,
                 neuroml_file,
                 target,
                 sim_time=1000,
                 dt=0.05,
                 simulator='jNeuroML',
                 generate_dir = './',
                 cleanup = True,
                 nml_doc = None):

        self.sim_time = sim_time
        self.dt = dt
        self.simulator = simulator
        self.generate_dir = generate_dir if generate_dir.endswith('/') else generate_dir+'/'

        self.reference = reference
        self.target = target
        self.neuroml_file = neuroml_file
        self.nml_doc = nml_doc
        self.cleanup = cleanup

        self.already_run = False

    def show(self):
        """
        Plot the result of the simulation once it's been intialized
        """

        from matplotlib import pyplot as plt

        if self.already_run:

            for ref in self.volts.keys():

                plt.plot(self.t, self.volts[ref], label=ref)
                plt.title("Simulation voltage vs time")
                plt.legend()
                plt.xlabel("Time [ms]")
                plt.ylabel("Voltage [mV]")

        else:
            pynml.print_comment("First you have to 'go()' the simulation.", True)
        plt.show()

    def go(self):

        lems_file_name = 'LEMS_%s.xml'%(self.reference)

        generate_lems_file_for_neuroml(self.reference,
                                       self.neuroml_file,
                                       self.target,
                                       self.sim_time,
                                       self.dt,
                                       lems_file_name = lems_file_name,
                                       target_dir = self.generate_dir,
                                       nml_doc = self.nml_doc)

        pynml.print_comment_v("Running a simulation of %s ms with timestep %s ms: %s"%(self.sim_time, self.dt, lems_file_name))

        self.already_run = True

        start = time.time()
        if self.simulator == 'jNeuroML':
            results = pynml.run_lems_with_jneuroml(lems_file_name,
                                                   nogui=True,
                                                   load_saved_data=True,
                                                   plot=False,
                                                   exec_in_dir = self.generate_dir,
                                                   verbose=False,
                                                   cleanup=self.cleanup)
        elif self.simulator == 'jNeuroML_NEURON':
            results = pynml.run_lems_with_jneuroml_neuron(lems_file_name,
                                                          nogui=True,
                                                          load_saved_data=True,
                                                          plot=False,
                                                          exec_in_dir = self.generate_dir,
                                                          verbose=False,
                                                          cleanup=self.cleanup)
        else:
            pynml.print_comment_v('Unsupported simulator: %s'%self.simulator)
            exit()

        secs = time.time()-start

        pynml.print_comment_v("Ran simulation in %s in %f seconds (%f mins)\n\n"%(self.simulator, secs, secs/60.0))

        self.t = [t*1000 for t in results['t']]

        self.volts = {}

        for key in results.keys():
            if key != 't':
                self.volts[key] = [v*1000 for v in results[key]]

if __name__ == '__main__':

    sim_time = 700
    dt = 0.05

    if len(sys.argv) == 2 and sys.argv[1] == '-net':

        sim = NeuroMLSimulation('TestNet',
                                '../../examples/test_data/simplenet.nml',
                                'simplenet',
                                sim_time,
                                dt,
                                'jNeuroML',
                                'temp/')
        sim.go()
        sim.show()

    else:

        sim = NeuroMLSimulation('TestHH',
                                '../../examples/test_data/HHCellNetwork.net.nml',
                                'HHCellNetwork',
                                sim_time,
                                dt,
                                'jNeuroML',
                                'temp')
        sim.go()
        sim.show()


# Author: Tom Dupre la Tour <tom.dupre-la-tour@m4x.org>
#
# License: BSD 3 clause

import numpy as np
import scipy.sparse as sp

from sklearn.utils.seq_dataset import ArrayDataset, CSRDataset
from sklearn.datasets import load_iris

from numpy.testing import assert_array_equal
from nose.tools import assert_equal

iris = load_iris()
X = iris.data.astype(np.float64)
y = iris.target.astype(np.float64)
X_csr = sp.csr_matrix(X)
sample_weight = np.arange(y.size, dtype=np.float64)

def assert_csr_equal(X, Y):
    X.eliminate_zeros()
    Y.eliminate_zeros()
    assert_equal(X.shape[0], Y.shape[0])
    assert_equal(X.shape[1], Y.shape[1])
    assert_array_equal(X.data, Y.data)
    assert_array_equal(X.indices, Y.indices)
    assert_array_equal(X.indptr, Y.indptr)

def test_seq_dataset():
    dataset1 = ArrayDataset(X, y, sample_weight, seed=42)
    dataset2 = CSRDataset(X_csr.data, X_csr.indptr, X_csr.indices,
                          y, sample_weight, seed=42)

    for dataset in (dataset1, dataset2):
        for i in range(5):
            # next sample
            xi_, yi, swi, idx = dataset._next_py()
            xi = sp.csr_matrix((xi_), shape=(1, X.shape[1]))

            assert_csr_equal(xi, X_csr[idx])
            assert_equal(yi, y[idx])
            assert_equal(swi, sample_weight[idx])

            # random sample
            xi_, yi, swi, idx = dataset._random_py()
            xi = sp.csr_matrix((xi_), shape=(1, X.shape[1]))

            assert_csr_equal(xi, X_csr[idx])
            assert_equal(yi, y[idx])
            assert_equal(swi, sample_weight[idx])

def test_seq_dataset_shuffle():
    dataset1 = ArrayDataset(X, y, sample_weight, seed=42)
    dataset2 = CSRDataset(X_csr.data, X_csr.indptr, X_csr.indices,
                          y, sample_weight, seed=42)

    # not shuffled
    for i in range(5):
        _, _, _, idx1 = dataset1._next_py()
        _, _, _, idx2 = dataset2._next_py()
        assert_equal(idx1, i)
        assert_equal(idx2, i)

    for i in range(5):
        _, _, _, idx1 = dataset1._random_py()
        _, _, _, idx2 = dataset2._random_py()
        assert_equal(idx1, idx2)

    seed = 77
    dataset1._shuffle_py(seed)
    dataset2._shuffle_py(seed)

    for i in range(5):
        _, _, _, idx1 = dataset1._next_py()
        _, _, _, idx2 = dataset2._next_py()
        assert_equal(idx1, idx2)

        _, _, _, idx1 = dataset1._random_py()
        _, _, _, idx2 = dataset2._random_py()
        assert_equal(idx1, idx2)


#  Copyright 2015-present The Scikit Flow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

import random

from tensorflow.contrib.learn.python import learn
from tensorflow.contrib.learn.python.learn import datasets
from tensorflow.contrib.learn.python.learn.estimators._sklearn import accuracy_score
from tensorflow.contrib.learn.python.learn.estimators._sklearn import train_test_split

class CustomDecayTest(tf.test.TestCase):

  def testIrisExponentialDecay(self):
    random.seed(42)

    iris = datasets.load_iris()
    X_train, X_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        test_size=0.2,
                                                        random_state=42)

    # setup exponential decay function
    def exp_decay(global_step):
      return tf.train.exponential_decay(learning_rate=0.1,
                                        global_step=global_step,
                                        decay_steps=100,
                                        decay_rate=0.001)

    classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
                                               n_classes=3,
                                               steps=500,
                                               learning_rate=exp_decay)
    classifier.fit(X_train, y_train)
    score = accuracy_score(y_test, classifier.predict(X_test))

    self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))

if __name__ == "__main__":
  tf.test.main()


#!usr/bin/python

# Copyright 2016, Rohan Dandage <rraadd_8@hotmail.com,rohan@igib.in>
# This program is distributed under General Public License v. 3.

"""
================================
``io_data_files``
================================
"""
import sys
import pandas as pd
from os.path import exists,basename,abspath,dirname,expanduser
import logging
from glob import glob
import numpy as np
from dms2dfe.lib.io_seq_files import get_fsta_feats
logging.basicConfig(format='[%(asctime)s] %(levelname)s\tfrom %(filename)s in %(funcName)s(..): %(message)s',level=logging.DEBUG) # filename=cfg_xls_fh+'.log'

import pickle

## DEFS
def is_cfg_ok(cfg_dh,cfgs) :
    """
    Checks if the required files are present in given directory.

    :param cfg_dh: path to directory.
    :param cfgs: list of names of files.
    """
    cfg_dh_cfgs=glob(cfg_dh+"/*")
    cfg_dh_cfgs=[basename(cfg_dh_cfg) for cfg_dh_cfg in cfg_dh_cfgs]
    for cfg in cfgs :   # check if required sheets are present
        if not cfg in cfg_dh_cfgs :
            logging.error("%s does not exist" % cfg)
            return False
            break
    return True

def auto_find_missing_paths(prj_dh):
    """
    Finds the missing paths in the configuration given in cfg/ directory

    :param prj_dh: path to the project directory
    """
    info=pd.read_csv(prj_dh+"/cfg/info")
    info_path_vars=[varn for varn in info['varname'] if ("_fh" in varn) or ("_dh" in varn)]
    info=info.set_index("varname")
    #find pdb_fh and fsta_fh in prj_dh
    if pd.isnull(info.loc["pdb_fh","input"]):
        try:
            info.loc["pdb_fh","input"]=glob("%s/*.pdb" % prj_dh)[0]
        except:
            logging.error("can not find .pdb file")
    if pd.isnull(info.loc["fsta_fh","input"]):
        try:
            fsta_fhs=glob("%s/*.fasta" % prj_dh)
            for fsta_fh in fsta_fhs:
                if not (('prt' in fsta_fh) or ('_cctmr1.' in fsta_fh)):
                    info.loc["fsta_fh","input"]=fsta_fh
                    break
        except:
            logging.error("could not find .fasta file")
    info_paths=[info.loc[info_path_var,"input"] for info_path_var in info_path_vars]
    info.reset_index().to_csv(prj_dh+"/cfg/info",index=False)
    # if any(pd.isnull(info_paths)):
    info_paths_missing=[v for v in info_path_vars if (pd.isnull(info.loc[v,"input"]) and info.loc[v,"default"])]
    if len(info_paths_missing)>0:
        logging.error("Values for following variables are missing in 'project_dir/cfg/info' file.")
        # print [p for p in info_paths if pd.isnull(p)]
        print info_paths_missing
        sys.exit()

def get_raw_input(info,var):
    """
    Get intearactive inputs from user

    :param info: dict, with information about experiment
    :param var: variable whose value is obtained from interactive shell
    """
    # from dms2dfe.lib.io_dfs import set_index
    # info=set_index(info,'var')
    val=raw_input("%s: %s (default: %s) =" % (var,info.loc[var, "description"],info.loc[var, "default"]))
    return val

from dms2dfe.lib.io_seq_files import cctmr_fasta2ref_fasta
def info2src(prj_dh):
    """
    This converts `.csv` configuration file to `.py` source file saved in `/tmp/`.

    :param prj_dh: path to project directory
    """
    import subprocess
    from dms2dfe.lib.io_seq_files import fasta_nts2prt

    csv2src("%s/../cfg/info" % abspath(dirname(__file__)),"%s/../tmp/info.py" % (abspath(dirname(__file__))))
    auto_find_missing_paths(prj_dh)
    info=pd.read_csv(prj_dh+"/cfg/info")
    # info=auto_find_missing_paths(prj_dh)
    info_path_vars=[varn for varn in info['varname'] if ("_fh" in varn) or ("_dh" in varn)]
    info=info.set_index("varname")

    # find still missing paths ones
    info_paths=[info.loc[info_path_var,"input"] for info_path_var in info_path_vars]
    for info_path_var,info_path in zip(info_path_vars,info_paths):
       # if not exists(info_path):
        if not ('bowtie' in info_path):
            if not exists(info_path):
                if info_path_var=='rscript_fh':
                    info_path = subprocess.check_output(["which", "Rscript"]).replace('\n','')
                    # print info_path
            while not exists(info_path):
                logging.error('Path to files do not exist. Include correct path in cfg/info. %s : %s' % (info_path_var,info_path))
                info_path=get_raw_input(info,info_path_var)
            info.loc[info_path_var,'input']=info_path

    if not pd.isnull(info.loc['cctmr','input']):
        cctmr=info.loc['cctmr','input']
        cctmr=[int("%s" % i) for i in cctmr.split(" ")]
        fsta_fh=cctmr_fasta2ref_fasta(info.loc['fsta_fh','input'],cctmr)
    else:
        fsta_fh=info.loc['fsta_fh','input']
    info.loc['prj_dh','input']=abspath(prj_dh)
    info.loc['fsta_id','input'],info.loc['fsta_seq','input'],info.loc['fsta_len','input']=get_fsta_feats(fsta_fh)
    host=info.loc['host','input']
    if pd.isnull(host):
        host=info.loc['host','default']
    info.loc['prt_seq','input']=fasta_nts2prt(fsta_fh,host=host).replace('*','X')
    info.reset_index().to_csv(prj_dh+"/cfg/info",index=False)
    csv2src(prj_dh+"/cfg/info","%s/../tmp/info.py" % (abspath(dirname(__file__))))
    csv2src(prj_dh+"/cfg/info",prj_dh+"/cfg/info.py")
    logging.info("configuration compiled: %s/cfg/info" % prj_dh)

def csv2src(csv_fh,src_fh):
    """
    This writes `.csv` to `.py` source file.

    :param csv_fh: path to input `.csv` file.
    :param src_fh: path to output `.py` source file.
    """
    info=pd.read_csv(csv_fh)
    info=info.set_index('varname')
    src_f=open(src_fh,'w')
    src_f.write("#!usr/bin/python\n")
    src_f.write("\n")
    src_f.write("# source file for dms2dfe's configuration \n")
    src_f.write("\n")

    for var in info.iterrows() :
        val=info['input'][var[0]]
        if pd.isnull(val):
            val=info['default'][var[0]]
        src_f.write("%s='%s' #%s\n" % (var[0],val,info["description"][var[0]]))
    src_f.close()

def raw_input2info(prj_dh,inputORdefault):
    """
    This writes configuration `.csv` file from `raw_input` from prompt.

    :param prj_dh: path to project directory.
    :param inputORdefault: column name "input" or "default".
    """
    info=pd.read_csv(prj_dh+"/cfg/info")
    info=info.set_index("varname",drop=True)
    for var in info.index.values:
        val=raw_input("%s (default: %s) =" % (info.loc[var, "description"],info.loc[var, "default"]))
        if not val=='':
            info.loc[var, inputORdefault]=val
    info.reset_index().to_csv("%s/cfg/info" % prj_dh, index=False)

def is_xls_ok(cfg_xls,cfg_xls_sheetnames_required) :
    """
    Checks if the required sheets are present in the configuration excel file.

    :param cfg_xls: path to configuration excel file
    """
    cfg_xls_sheetnames=cfg_xls.sheet_names
    cfg_xls_sheetnames= [str(x) for x in cfg_xls_sheetnames]# unicode to str

    for qry_sheet_namei in cfg_xls_sheetnames_required :   # check if required sheets are present
        #qry_sheet_namei=str(qry_sheet_namei)
        if not qry_sheet_namei in cfg_xls_sheetnames :
            logging.error("pipeline : sheetname '%s' does not exist" % qry_sheet_namei)
            return False
            break
    return True

def is_info_ok(xls_fh):
    """
    This checks the sanity of info sheet in the configuration excel file.
    For example if the files exists or not.

    :param cfg_xls: path to configuration excel file
    """
    info=pd.read_excel(xls_fh,'info')
    info_path_vars=[varn for varn in info['varname'] if ("_fh" in varn) or ("_dh" in varn)]
    info=info.set_index("varname")
    info_paths=[info.loc[info_path_var,"input"] for info_path_var in info_path_vars]
    for info_path in info_paths:
        if not pd.isnull(info_path):
            if not exists(info_path):
                return False #(info_path_vars[info_paths.index(info_path)],info_path)
                break
    return True

def xls2h5(cfg_xls,cfg_h5,cfg_xls_sheetnames_required) :
    """
    Converts configuration excel file to HDF5(h5) file.
    Here sheets in excel files are converted to groups in HDF5 file.

    :param cfg_xls: path to configuration excel file
    """
    for qry_sheet_namei in  cfg_xls_sheetnames_required:
        qry_sheet_df=cfg_xls.parse(qry_sheet_namei)
        qry_sheet_df=qry_sheet_df.astype(str) # suppress unicode error
        qry_sheet_df.columns=[col.replace(" ","_") for col in qry_sheet_df.columns]
        cfg_h5.put("cfg/"+qry_sheet_namei,convert2h5form(qry_sheet_df), format='table', data_columns=True)
    return cfg_h5

def xls2csvs(cfg_xls,cfg_xls_sheetnames_required,output_dh):
    """
    Converts configuration excel file to HDF5(h5) file.
    Here sheets in excel files are converted to groups in HDF5 file.

    :param cfg_xls: path to configuration excel file
    """
    for qry_sheet_namei in  cfg_xls_sheetnames_required:
        qry_sheet_df=cfg_xls.parse(qry_sheet_namei)
        qry_sheet_df=qry_sheet_df.astype(str) # suppress unicode error
        qry_sheet_df.to_csv("%s/%s" % (output_dh,qry_sheet_namei))
#         print "%s/%s" % (output_dh,qry_sheet_namei)

def convert2h5form(df):
    """
    Convert dataframe compatible to  Hdf5 format

    :param df: pandas dataframe
    """
    from dms2dfe.lib.io_strs import convertstr2format
    df.columns=[convertstr2format(col,"^[a-zA-Z0-9_]*$") for col in df.columns.tolist()]
    return df

def csvs2h5(dh,sub_dh_list,fn_list,output_dh,cfg_h5):
    """
    This converts the csv files to tables in HDF5.

    :param dh: path to the directory with csv files
    :param fn_list: list of filenames of the csv files
    """
    for fn in fn_list:
        for sub_dh in sub_dh_list : # get aas or cds
            fh=output_dh+"/"+dh+"/"+sub_dh+"/"+fn+""
            df=pd.read_csv(fh) # get mat to df
            df=df.loc[:,[col.replace(" ","_") for col in list(df.columns) if not (('index' in col) or ('Unnamed' in col)) ]]
            exec("cfg_h5.put('%s/%s/%s',df, format='table', data_columns=True)" % (dh,sub_dh,str(fn)),locals(), globals()) # store the otpts in h5 eg. cds/N/lbl
            # print("cfg_h5.put('%s/%s/%s',df.convert_objects(), format='table', data_columns=True)" % (dh,sub_dh,str(fn))) # store the otpts in h5 eg. cds/N/lbl

def csvs2h5(dh,sub_dh_list,fn_list):
    """
    This converts csvs into HDF5 tables.

    :param dh: path to the directory with csv files
    :param fn_list: list of filenames of the csv files
    """
    for fn in fn_list:
        for sub_dh in sub_dh_list : # get aas or cds
            fh=output_dh+"/"+dh+"/"+sub_dh+"/"+fn+""
            key=dh+"/"+sub_dh+"/"+fn
            if (exists(fh)) and (key in cfg_h5):
                df=pd.read_csv(fh) # get mat to df
                key=key+"2"
                cfg_h5.put(key,df.convert_objects(), format='table', data_columns=True) # store the otpts in h5 eg. cds/N/lbl

#mut_lbl_fit_comparison
def getusable_lbls_list(prj_dh):
    """
    This detects the samples that can be processed.

    :param prj_dh: path to project directory.
    :returns lbls_list: list of names of samples that can be processed.
    """
    lbls=pd.read_csv(prj_dh+'/cfg/lbls')
    lbls=lbls.set_index('varname')
    lbls_list=[]
    #data_lbl cols: NiA mutids NiS NiN NiNcut NiNcutlog NiScut NiScutlog NiAcut NiAcutlog
    for lbli,lbl in lbls.iterrows() :
        # print "%s/data_lbl/%s/%s" % (prj_dh,'aas',str(lbli))
        if (not exists("%s/data_lbl/%s/%s" % (prj_dh,'aas',str(lbli)))):
            fh_1=expanduser(str(lbl['fhs_1']))
            lbl_mat_mut_cds_fh=[fh for fh in glob(fh_1+"*") if '.mat_mut_cds' in fh]
            if len(lbl_mat_mut_cds_fh)!=0:
                lbl_mat_mut_cds_fh=lbl_mat_mut_cds_fh[0]
                lbls_list.append([lbli,lbl_mat_mut_cds_fh])
            else :
                fh_1="%s/data_mutmat/%s" % (prj_dh,basename(fh_1))
                # print fh_1
                lbl_mat_mut_cds_fh=[fh for fh in glob(fh_1+"*") if '.mat_mut_cds' in fh]
                if len(lbl_mat_mut_cds_fh)!=0:
                    lbl_mat_mut_cds_fh=lbl_mat_mut_cds_fh[0]
                    lbls_list.append([lbli,lbl_mat_mut_cds_fh])
                else:
                    logging.warning("can not find: %s" % fh_1)
        # else:
            # logging.info("already processed: %s" % (str(lbli)))
    return lbls_list

def getusable_fits_list(prj_dh,data_fit_dh='data_fit'):
    """
    This gets the list of samples that can be processed for fitness estimations.

    :param prj_dh: path to project directory.
    :returns fits_pairs_list: list of tuples with names of input and selected samples.
    """
    if exists('%s/cfg/fit'% (prj_dh)):
        fits=pd.read_csv(prj_dh+'/cfg/fit')

        if "Unnamed: 0" in fits.columns:
            fits=fits.drop("Unnamed: 0", axis=1)
        fits_pairs_list=[]
        sel_cols=[col for col in fits.columns.tolist() if "sel_" in col]
        for pairi in fits.index.values :
            unsel_lbl=fits.loc[pairi,"unsel"]
            sels=list(fits.loc[pairi,sel_cols])
            # print sels
            for sel_lbl in sels :
                if not pd.isnull(sel_lbl):
                    fit_lbl=sel_lbl+"_WRT_"+unsel_lbl
                    if (not exists("%s/%s/%s/%s" % (prj_dh,data_fit_dh,'aas',fit_lbl))):
                        fits_pairs_list.append([unsel_lbl,sel_lbl])
                    else :
                        logging.info("already processed: %s" % (fit_lbl))
        return fits_pairs_list
    else:
        logging.warning("ana3_mutmat2fit : getusable_fits_list : not fits in cfg/fit")
        return []

def getusable_comparison_list(prj_dh):
    """
    This converts the table of tests and controls in configuration file into tuples of test and control.

    :param prj_dh: path to project directory.
    """
    comparisons=pd.read_csv(prj_dh+'/cfg/comparison')
    comparisons=comparisons.set_index('ctrl')
    comparison_list=[]
    for ctrl,row in comparisons.iterrows() :
        row=row[~row.isnull()]
        for test in row[0:] :
            comparison_list.append([ctrl,test])
    return  comparison_list

def to_pkl(data,fh):
    """
    Saves a dict in pkl format

    :param data: dict, containing data
    :param fh: path to the output pkl file
    """
    if not fh is None:
        with open(fh, 'wb') as f:
            pickle.dump(data, f, -1)

def read_pkl(fh):
    """
    Reads a file in pkl format

    :param fh: path to the pkl file
    :returns data: dict, containing data
    """
    with open(fh,'rb') as f:
        return pickle.load(f)