#! /usr/bin/env python
"""
Methods to plot data defined on Landlab grids.

Plotting functions
++++++++++++++++++

.. autosummary::
    :toctree: generated/

    ~landlab.plot.imshow.imshow_grid
    ~landlab.plot.imshow.imshow_grid_at_cell
    ~landlab.plot.imshow.imshow_grid_at_node
"""

import numpy as np
import inspect
from landlab.field.scalar_data_fields import FieldError
try:
    import matplotlib.pyplot as plt
except ImportError:
    import warnings
    warnings.warn('matplotlib not found', ImportWarning)
from landlab.grid import CLOSED_BOUNDARY
from landlab.grid.raster import RasterModelGrid
from landlab.grid.voronoi import VoronoiDelaunayGrid
from landlab.utils.decorators import deprecated

def imshow_grid_at_node(grid, values, **kwds):
    """Prepare a map view of data over all nodes in the grid.

    Data is plotted as cells shaded with the value at the node at its center.
    Outer edges of perimeter cells are extrapolated. Closed elements are
    colored uniformly (default black, overridden with kwd 'color_for_closed');
    other open boundary nodes get their actual values.

    *values* can be a field name, a regular array, or a masked array. If a
    masked array is provided, masked entries will be treated as if they were
    Landlab CLOSED_BOUNDARYs. Used together with the color_at_closed=None
    keyword (i.e., "transparent"), this can allow for construction of overlay
    layers in a figure (e.g., only defining values in a river network, and
    overlaying it on another landscape).

    Use matplotlib functions like xlim, ylim to modify your plot after calling
    :func:`imshow_grid`, as desired.

    This function happily works with both regular and irregular grids.

    Construction ::

        imshow_grid_at_node(grid, values, plot_name=None, var_name=None,
                            var_units=None, grid_units=None,
                            symmetric_cbar=False, cmap='pink',
                            limits=(values.min(), values.max()),
                            vmin=values.min(), vmax=values.max(),
                            allow_colorbar=True,
                            norm=[linear], shrink=1.,
                            color_for_closed='black',
                            color_for_background=None,
                            show_elements=False, output=None)

    Parameters
    ----------
    grid : ModelGrid
        Grid containing the field to plot, or describing the geometry of the
        provided array.
    values : array_like, masked_array, or str
        Node values, or a field name as a string from which to draw the data.
    plot_name : str, optional
        String to put as the plot title.
    var_name : str, optional
        Variable name, to use as a colorbar label.
    var_units : str, optional
        Units for the variable being plotted, for the colorbar.
    grid_units : tuple of str, optional
        Units for y, and x dimensions. If None, component will look to the
        gri property `axis_units` for this information. If no units are
        specified there, no entry is made.
    symmetric_cbar : bool
        Make the colormap symetric about 0.
    cmap : str
        Name of a colormap
    limits : tuple of float
        Minimum and maximum of the colorbar.
    vmin, vmax: floats
        Alternatives to limits.
    allow_colorbar : bool
        If True, include the colorbar.
    colorbar_label : str or None
        The string with which to label the colorbar.
    norm : matplotlib.colors.Normalize
        The normalizing object which scales data, typically into the interval
        [0, 1]. Ignore in most cases.
    shrink : float
        Fraction by which to shrink the colorbar.
    color_for_closed : str or None
        Color to use for closed nodes (default 'black'). If None, closed
        (or masked) nodes will be transparent.
    color_for_background : color str or other color declaration, or None
        Color to use for closed elements (default None). If None, the
        background will be transparent, and appear white.
    show_elements : bool
        If True, and grid is a Voronoi, the faces will be plotted in black
        along with just the colour of the cell, defining the cell outlines
        (defaults False).
    output : None, string, or bool
        If None (or False), the image is sent to the imaging buffer to await
        an explicit call to show() or savefig() from outside this function.
        If a string, the string should be the path to a save location, and the
        filename (with file extension). The function will then call
        plt.savefig([string]) itself. If True, the function will call
        plt.show() itself once plotting is complete.
    """
    if isinstance(values, str):
        values_at_node = grid.at_node[values]
    else:
        values_at_node = values

    if values_at_node.size != grid.number_of_nodes:
        raise ValueError('number of values does not match number of nodes')

    values_at_node = np.ma.masked_where(
        grid.status_at_node == CLOSED_BOUNDARY, values_at_node)

    try:
        shape = grid.shape
    except AttributeError:
        shape = (-1, )

    _imshow_grid_values(grid, values_at_node.reshape(shape), **kwds)

    if isinstance(values, str):
        plt.title(values)

@deprecated(use='imshow_grid_at_node', version='0.5')
def imshow_node_grid(grid, values, **kwds):
    imshow_grid_at_node(grid, values, **kwds)

def imshow_grid_at_cell(grid, values, **kwds):
    """Map view of grid data over all grid cells.

    Prepares a map view of data over all cells in the grid.
    Method can take any of the same ``**kwds`` as :func:`imshow_grid_at_node`.

    Construction ::

        imshow_grid_at_cell(grid, values, plot_name=None, var_name=None,
                            var_units=None, grid_units=None,
                            symmetric_cbar=False, cmap='pink',
                            limits=(values.min(), values.max()),
                            vmin=values.min(), vmax=values.max(),
                            allow_colorbar=True, colorbar_label=None,
                            norm=[linear], shrink=1.,
                            color_for_closed='black',
                            color_for_background=None,
                            show_elements=False, output=None)

    Parameters
    ----------
    grid : ModelGrid
        Grid containing the field to plot, or describing the geometry of the
        provided array.
    values : array_like, masked_array, or str
        Values at the cells on the grid. Alternatively, can be a field name
        (string) from which to draw the data from the grid.
    plot_name : str, optional
        String to put as the plot title.
    var_name : str, optional
        Variable name, to use as a colorbar label.
    var_units : str, optional
        Units for the variable being plotted, for the colorbar.
    grid_units : tuple of str, optional
        Units for y, and x dimensions. If None, component will look to the
        gri property `axis_units` for this information. If no units are
        specified there, no entry is made.
    symmetric_cbar : bool
        Make the colormap symetric about 0.
    cmap : str
        Name of a colormap
    limits : tuple of float
        Minimum and maximum of the colorbar.
    vmin, vmax: floats
        Alternatives to limits.
    allow_colorbar : bool
        If True, include the colorbar.
    colorbar_label : str or None
        The string with which to label the colorbar.
    norm : matplotlib.colors.Normalize
        The normalizing object which scales data, typically into the interval
        [0, 1]. Ignore in most cases.
    shrink : float
        Fraction by which to shrink the colorbar.
    color_for_closed : str or None
        Color to use for closed elements (default 'black'). If None, closed
        (or masked) elements will be transparent.
    color_for_background : color str or other color declaration, or None
        Color to use for closed elements (default None). If None, the
        background will be transparent, and appear white.
    show_elements : bool
        If True, and grid is a Voronoi, the faces will be plotted in black
        along with just the colour of the cell, defining the cell outlines
        (defaults False).
    output : None, string, or bool
        If None (or False), the image is sent to the imaging buffer to await
        an explicit call to show() or savefig() from outside this function.
        If a string, the string should be the path to a save location, and the
        filename (with file extension). The function will then call
        plt.savefig([string]) itself. If True, the function will call
        plt.show() itself once plotting is complete.

    Raises
    ------
    ValueError
        If input grid is not uniform rectilinear.
    """
    if isinstance(values, str):
        try:
            values_at_cell = grid.at_cell[values]
        except FieldError:
            values_at_cell = grid.at_node[values]
    else:
        values_at_cell = values

    if values_at_cell.size == grid.number_of_nodes:
        values_at_cell = values_at_cell[grid.node_at_cell]

    if values_at_cell.size != grid.number_of_cells:
        raise ValueError('number of values must match number of cells or '
                         'number of nodes')

    values_at_cell = np.ma.asarray(values_at_cell)
    values_at_cell.mask = True
    values_at_cell.mask[grid.core_cells] = False

    myimage = _imshow_grid_values(grid,
                                  values_at_cell.reshape(grid.cell_grid_shape),
                                  **kwds)

    if isinstance(values, str):
        plt.title(values)

    return myimage

@deprecated(use='imshow_grid_at_cell', version='0.5')
def imshow_cell_grid(grid, values, **kwds):
    imshow_grid_at_cell(grid, values, **kwds)

def _imshow_grid_values(grid, values, plot_name=None, var_name=None,
                        var_units=None, grid_units=(None, None),
                        symmetric_cbar=False, cmap='pink', limits=None,
                        colorbar_label = None,
                        allow_colorbar=True, vmin=None, vmax=None,
                        norm=None, shrink=1., color_for_closed='black',
                        color_for_background=None, show_elements=False,
                        output=None):

    gridtypes = inspect.getmro(grid.__class__)

    cmap = plt.get_cmap(cmap)
    if color_for_closed is not None:
        cmap.set_bad(color=color_for_closed)
    else:
        cmap.set_bad(alpha=0.)

    if isinstance(grid, RasterModelGrid):
        if values.ndim != 2:
            raise ValueError('values must have ndim == 2')

        y = np.arange(values.shape[0] + 1) * grid.dy - grid.dy * .5
        x = np.arange(values.shape[1] + 1) * grid.dx - grid.dx * .5

        kwds = dict(cmap=cmap)
        (kwds['vmin'], kwds['vmax']) = (values.min(), values.max())
        if (limits is None) and ((vmin is None) and (vmax is None)):
            if symmetric_cbar:
                (var_min, var_max) = (values.min(), values.max())
                limit = max(abs(var_min), abs(var_max))
                (kwds['vmin'], kwds['vmax']) = (- limit, limit)
        elif limits is not None:
            (kwds['vmin'], kwds['vmax']) = (limits[0], limits[1])
        else:
            if vmin is not None:
                kwds['vmin'] = vmin
            if vmax is not None:
                kwds['vmax'] = vmax

        if np.isclose(grid.dx, grid.dy):
            if values.size == grid.number_of_nodes:
                myimage = plt.imshow(
                    values.reshape(grid.shape), origin='lower',
                    extent=(x[0], x[-1], y[0], y[-1]), **kwds)
            else:  # this is a cell grid, and has been reshaped already...
                myimage = plt.imshow(values, origin='lower',
                                     extent=(x[0], x[-1], y[0], y[-1]), **kwds)
        myimage = plt.pcolormesh(x, y, values, **kwds)

        plt.gca().set_aspect(1.)
        plt.autoscale(tight=True)

        if allow_colorbar:
            cb = plt.colorbar(norm=norm, shrink=shrink)
            if colorbar_label:
                cb.set_label(colorbar_label)
    elif VoronoiDelaunayGrid in gridtypes:
        # This is still very much ad-hoc, and needs prettifying.
        # We should save the modifications needed to plot color all the way
        # to the diagram edge *into* the grid, for faster plotting.
        # (see http://stackoverflow.com/questions/20515554/...
        # colorize-voronoi-diagram)
        # (This technique is not implemented yet)
        from scipy.spatial import voronoi_plot_2d
        import matplotlib.colors as colors
        import matplotlib.cm as cmx
        cm = plt.get_cmap(cmap)

        if (limits is None) and ((vmin is None) and (vmax is None)):
            # only want to work with NOT CLOSED nodes
            open_nodes = grid.status_at_node != 4
            if symmetric_cbar:
                (var_min, var_max) = (values.flat[
                    open_nodes].min(), values.flat[open_nodes].max())
                limit = max(abs(var_min), abs(var_max))
                (vmin, vmax) = (- limit, limit)
            else:
                (vmin, vmax) = (values.flat[
                    open_nodes].min(), values.flat[open_nodes].max())
        elif limits is not None:
            (vmin, vmax) = (limits[0], limits[1])
        else:
            open_nodes = grid.status_at_node != 4
            if vmin is None:
                vmin = values.flat[open_nodes].min()
            if vmax is None:
                vmax = values.flat[open_nodes].max()

        cNorm = colors.Normalize(vmin, vmax)
        scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=cm)
        colorVal = scalarMap.to_rgba(values)

        if show_elements:
            myimage = voronoi_plot_2d(grid.vor, show_vertices=False,
                                      show_points=False)
        # show_points to be supported in scipy0.18, but harmless for now
        mycolors = (i for i in colorVal)
        for order in grid.vor.point_region:
            region = grid.vor.regions[order]
            colortouse = next(mycolors)
            if -1 not in region:
                polygon = [grid.vor.vertices[i] for i in region]
                plt.fill(*zip(*polygon), color=colortouse)

        plt.gca().set_aspect(1.)
        # plt.autoscale(tight=True)
        # Tempting though it is to move the boundary outboard of the outermost
        # nodes (e.g., to the outermost corners), this is a bad idea, as the
        # outermost cells tend to have highly elongated shapes which make the
        # plot look stupid
        plt.xlim((np.min(grid.node_x), np.max(grid.node_x)))
        plt.ylim((np.min(grid.node_y), np.max(grid.node_y)))

        scalarMap.set_array(values)
        if allow_colorbar:
            cb = plt.colorbar(scalarMap, shrink=shrink)

    if grid_units[1] is None and grid_units[0] is None:
        grid_units = grid.axis_units
        if grid_units[1] == '-' and grid_units[0] == '-':
            plt.xlabel('X')
            plt.ylabel('Y')
        else:
            plt.xlabel('X (%s)' % grid_units[1])
            plt.ylabel('Y (%s)' % grid_units[0])
    else:
        plt.xlabel('X (%s)' % grid_units[1])
        plt.ylabel('Y (%s)' % grid_units[0])

    if plot_name is not None:
        plt.title('%s' % (plot_name))

    if var_name is not None or var_units is not None:
        if var_name is not None:
            assert type(var_name) is str
            if var_units is not None:
                assert type(var_units) is str
                colorbar_label = var_name + ' (' + var_units + ')'
            else:
                colorbar_label = var_name
        else:
            assert type(var_units) is str
            colorbar_label = '(' + var_units + ')'
        assert type(colorbar_label) is str
        assert allow_colorbar
        cb.set_label(colorbar_label)

    if color_for_background is not None:
        plt.gca().set_axis_bgcolor(color_for_background)

    if output is not None:
        if type(output) is str:
            plt.savefig(output)
            plt.clf()
        elif output:
            plt.show()

def imshow_grid(grid, values, **kwds):
    """Prepare a map view of data over all nodes or cells in the grid.

    Data is plotted as colored cells. If at='node', the surrounding cell is
    shaded with the value at the node at its center. If at='cell', the cell
    is shaded with its own value. Outer edges of perimeter cells are
    extrapolated. Closed elements are colored uniformly (default black,
    overridden with kwd 'color_for_closed'); other open boundary nodes get
    their actual values.

    *values* can be a field name, a regular array, or a masked array. If a
    masked array is provided, masked entries will be treated as if they were
    Landlab CLOSED_BOUNDARYs. Used together with the color_at_closed=None
    keyword (i.e., "transparent"), this can allow for construction of overlay
    layers in a figure (e.g., only defining values in a river network, and
    overlaying it on another landscape).

    Use matplotlib functions like xlim, ylim to modify your plot after calling
    :func:`imshow_grid`, as desired.

    This function happily works with both regular and irregular grids.

    Construction ::

        imshow_grid(grid, values, plot_name=None, var_name=None,
                    var_units=None, grid_units=None,
                    symmetric_cbar=False, cmap='pink',
                    limits=(values.min(), values.max()),
                    vmin=values.min(), vmax=values.max(),
                    allow_colorbar=True, colorbar_label=None,
                    norm=[linear], shrink=1.,
                    color_for_closed='black',
                    color_for_background=None,
                    show_elements=False)

    Parameters
    ----------
    grid : ModelGrid
        Grid containing the field to plot, or describing the geometry of the
        provided array.
    values : array_like, masked_array, or str
        Node or cell values, or a field name as a string from which to draw
        the data.
    at : str, {'node', 'cell'}
        Tells plotter where values are defined.
    plot_name : str, optional
        String to put as the plot title.
    var_name : str, optional
        Variable name, to use as a colorbar label.
    var_units : str, optional
        Units for the variable being plotted, for the colorbar.
    grid_units : tuple of str, optional
        Units for y, and x dimensions. If None, component will look to the
        gri property `axis_units` for this information. If no units are
        specified there, no entry is made.
    symmetric_cbar : bool
        Make the colormap symetric about 0.
    cmap : str
        Name of a colormap
    limits : tuple of float
        Minimum and maximum of the colorbar.
    vmin, vmax: floats
        Alternatives to limits.
    allow_colorbar : bool
        If True, include the colorbar.
    colorbar_label : str or None
        The string with which to label the colorbar.
    norm : matplotlib.colors.Normalize
        The normalizing object which scales data, typically into the interval
        [0, 1]. Ignore in most cases.
    shrink : float
        Fraction by which to shrink the colorbar.
    color_for_closed : str or None
        Color to use for closed elements (default 'black'). If None, closed
        (or masked) elements will be transparent.
    color_for_background : color str or other color declaration, or None
        Color to use for closed elements (default None). If None, the
        background will be transparent, and appear white.
    show_elements : bool
        If True, and grid is a Voronoi, the faces will be plotted in black
        along with just the colour of the cell, defining the cell outlines
        (defaults False).
    output : None, string, or bool
        If None (or False), the image is sent to the imaging buffer to await
        an explicit call to show() or savefig() from outside this function.
        If a string, the string should be the path to a save location, and the
        filename (with file extension). The function will then call
        plt.savefig([string]) itself. If True, the function will call
        plt.show() itself once plotting is complete.
    """
    show = kwds.pop('show', False)
    values_at = kwds.pop('values_at', 'node')
    values_at = kwds.pop('at', values_at)

    if isinstance(values, str):
        values = grid.field_values(values_at, values)

    if isinstance(values, str):
        values = grid.field_values(values_at, values)

    if values_at == 'node':
        imshow_grid_at_node(grid, values, **kwds)
    elif values_at == 'cell':
        imshow_grid_at_cell(grid, values, **kwds)
    else:
        raise TypeError('value location %s not understood' % values_at)

    # retained for backwards compatibility:
    if show:
        plt.show()


"""
=========================================
SGD: Maximum margin separating hyperplane
=========================================

Plot the maximum margin separating hyperplane within a two-class
separable dataset using a linear Support Vector Machines classifier
trained using SGD.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import SGDClassifier
from sklearn.datasets.samples_generator import make_blobs

# we create 50 separable points
X, Y = make_blobs(n_samples=50, centers=2, random_state=0, cluster_std=0.60)

# fit the model
clf = SGDClassifier(loss="hinge", alpha=0.01, n_iter=200, fit_intercept=True)
clf.fit(X, Y)

# plot the line, the points, and the nearest vectors to the plane
xx = np.linspace(-1, 5, 10)
yy = np.linspace(-1, 5, 10)

X1, X2 = np.meshgrid(xx, yy)
Z = np.empty(X1.shape)
for (i, j), val in np.ndenumerate(X1):
    x1 = val
    x2 = X2[i, j]
    p = clf.decision_function([x1, x2])
    Z[i, j] = p[0]
levels = [-1.0, 0.0, 1.0]
linestyles = ['dashed', 'solid', 'dashed']
colors = 'k'
plt.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles)
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)

plt.axis('tight')
plt.show()


# -*- coding: utf-8 -*-

"""
Tests that quoting specifications are properly handled
during parsing for all of the parsers defined in parsers.py
"""

import csv
import pandas.util.testing as tm

from pandas import DataFrame
from pandas.compat import PY3, StringIO, u

class QuotingTests(object):

    def test_bad_quote_char(self):
        data = '1,2,3'

        # Python 2.x: "...must be an 1-character..."
        # Python 3.x: "...must be a 1-character..."
        msg = '"quotechar" must be a(n)? 1-character string'
        tm.assertRaisesRegexp(TypeError, msg, self.read_csv,
                              StringIO(data), quotechar='foo')

        msg = 'quotechar must be set if quoting enabled'
        tm.assertRaisesRegexp(TypeError, msg, self.read_csv,
                              StringIO(data), quotechar=None,
                              quoting=csv.QUOTE_MINIMAL)

        msg = '"quotechar" must be string, not int'
        tm.assertRaisesRegexp(TypeError, msg, self.read_csv,
                              StringIO(data), quotechar=2)

    def test_bad_quoting(self):
        data = '1,2,3'

        msg = '"quoting" must be an integer'
        tm.assertRaisesRegexp(TypeError, msg, self.read_csv,
                              StringIO(data), quoting='foo')

        # quoting must in the range [0, 3]
        msg = 'bad "quoting" value'
        tm.assertRaisesRegexp(TypeError, msg, self.read_csv,
                              StringIO(data), quoting=5)

    def test_quote_char_basic(self):
        data = 'a,b,c\n1,2,"cat"'
        expected = DataFrame([[1, 2, 'cat']],
                             columns=['a', 'b', 'c'])
        result = self.read_csv(StringIO(data), quotechar='"')
        tm.assert_frame_equal(result, expected)

    def test_quote_char_various(self):
        data = 'a,b,c\n1,2,"cat"'
        expected = DataFrame([[1, 2, 'cat']],
                             columns=['a', 'b', 'c'])
        quote_chars = ['~', '*', '%', '$', '@', 'P']

        for quote_char in quote_chars:
            new_data = data.replace('"', quote_char)
            result = self.read_csv(StringIO(new_data), quotechar=quote_char)
            tm.assert_frame_equal(result, expected)

    def test_null_quote_char(self):
        data = 'a,b,c\n1,2,3'

        # sanity checks
        msg = 'quotechar must be set if quoting enabled'

        tm.assertRaisesRegexp(TypeError, msg, self.read_csv,
                              StringIO(data), quotechar=None,
                              quoting=csv.QUOTE_MINIMAL)

        tm.assertRaisesRegexp(TypeError, msg, self.read_csv,
                              StringIO(data), quotechar='',
                              quoting=csv.QUOTE_MINIMAL)

        # no errors should be raised if quoting is None
        expected = DataFrame([[1, 2, 3]],
                             columns=['a', 'b', 'c'])

        result = self.read_csv(StringIO(data), quotechar=None,
                               quoting=csv.QUOTE_NONE)
        tm.assert_frame_equal(result, expected)

        result = self.read_csv(StringIO(data), quotechar='',
                               quoting=csv.QUOTE_NONE)
        tm.assert_frame_equal(result, expected)

    def test_quoting_various(self):
        data = '1,2,"foo"'
        cols = ['a', 'b', 'c']

        # QUOTE_MINIMAL and QUOTE_ALL apply only to
        # the CSV writer, so they should have no
        # special effect for the CSV reader
        expected = DataFrame([[1, 2, 'foo']], columns=cols)

        # test default (afterwards, arguments are all explicit)
        result = self.read_csv(StringIO(data), names=cols)
        tm.assert_frame_equal(result, expected)

        result = self.read_csv(StringIO(data), quotechar='"',
                               quoting=csv.QUOTE_MINIMAL, names=cols)
        tm.assert_frame_equal(result, expected)

        result = self.read_csv(StringIO(data), quotechar='"',
                               quoting=csv.QUOTE_ALL, names=cols)
        tm.assert_frame_equal(result, expected)

        # QUOTE_NONE tells the reader to do no special handling
        # of quote characters and leave them alone
        expected = DataFrame([[1, 2, '"foo"']], columns=cols)
        result = self.read_csv(StringIO(data), quotechar='"',
                               quoting=csv.QUOTE_NONE, names=cols)
        tm.assert_frame_equal(result, expected)

        # QUOTE_NONNUMERIC tells the reader to cast
        # all non-quoted fields to float
        expected = DataFrame([[1.0, 2.0, 'foo']], columns=cols)
        result = self.read_csv(StringIO(data), quotechar='"',
                               quoting=csv.QUOTE_NONNUMERIC,
                               names=cols)
        tm.assert_frame_equal(result, expected)

    def test_double_quote(self):
        data = 'a,b\n3,"4 "" 5"'

        expected = DataFrame([[3, '4 " 5']],
                             columns=['a', 'b'])
        result = self.read_csv(StringIO(data), quotechar='"',
                               doublequote=True)
        tm.assert_frame_equal(result, expected)

        expected = DataFrame([[3, '4 " 5"']],
                             columns=['a', 'b'])
        result = self.read_csv(StringIO(data), quotechar='"',
                               doublequote=False)
        tm.assert_frame_equal(result, expected)

    def test_quotechar_unicode(self):
        # See gh-14477
        data = 'a\n1'
        expected = DataFrame({'a': [1]})

        result = self.read_csv(StringIO(data), quotechar=u('"'))
        tm.assert_frame_equal(result, expected)

        # Compared to Python 3.x, Python 2.x does not handle unicode well.
        if PY3:
            result = self.read_csv(StringIO(data), quotechar=u('\u0394'))
            tm.assert_frame_equal(result, expected)


import numpy as np
from sklearn.utils.testing import (assert_allclose, assert_raises,
                                   assert_equal)
from sklearn.neighbors import KernelDensity, KDTree, NearestNeighbors
from sklearn.neighbors.ball_tree import kernel_norm
from sklearn.pipeline import make_pipeline
from sklearn.datasets import make_blobs
from sklearn.grid_search import GridSearchCV
from sklearn.preprocessing import StandardScaler

def compute_kernel_slow(Y, X, kernel, h):
    d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
    norm = kernel_norm(h, X.shape[1], kernel) / X.shape[0]

    if kernel == 'gaussian':
        return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
    elif kernel == 'tophat':
        return norm * (d < h).sum(-1)
    elif kernel == 'epanechnikov':
        return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
    elif kernel == 'exponential':
        return norm * (np.exp(-d / h)).sum(-1)
    elif kernel == 'linear':
        return norm * ((1 - d / h) * (d < h)).sum(-1)
    elif kernel == 'cosine':
        return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
    else:
        raise ValueError('kernel not recognized')

def test_kernel_density(n_samples=100, n_features=3):
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features)
    Y = rng.randn(n_samples, n_features)

    for kernel in ['gaussian', 'tophat', 'epanechnikov',
                   'exponential', 'linear', 'cosine']:
        for bandwidth in [0.01, 0.1, 1]:
            dens_true = compute_kernel_slow(Y, X, kernel, bandwidth)

            def check_results(kernel, bandwidth, atol, rtol):
                kde = KernelDensity(kernel=kernel, bandwidth=bandwidth,
                                    atol=atol, rtol=rtol)
                log_dens = kde.fit(X).score_samples(Y)
                assert_allclose(np.exp(log_dens), dens_true,
                                atol=atol, rtol=max(1E-7, rtol))
                assert_allclose(np.exp(kde.score(Y)),
                                np.prod(dens_true),
                                atol=atol, rtol=max(1E-7, rtol))

            for rtol in [0, 1E-5]:
                for atol in [1E-6, 1E-2]:
                    for breadth_first in (True, False):
                        yield (check_results, kernel, bandwidth, atol, rtol)

def test_kernel_density_sampling(n_samples=100, n_features=3):
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features)

    bandwidth = 0.2

    for kernel in ['gaussian', 'tophat']:
        # draw a tophat sample
        kde = KernelDensity(bandwidth, kernel=kernel).fit(X)
        samp = kde.sample(100)
        assert_equal(X.shape, samp.shape)

        # check that samples are in the right range
        nbrs = NearestNeighbors(n_neighbors=1).fit(X)
        dist, ind = nbrs.kneighbors(X, return_distance=True)

        if kernel == 'tophat':
            assert np.all(dist < bandwidth)
        elif kernel == 'gaussian':
            # 5 standard deviations is safe for 100 samples, but there's a
            # very small chance this test could fail.
            assert np.all(dist < 5 * bandwidth)

    # check unsupported kernels
    for kernel in ['epanechnikov', 'exponential', 'linear', 'cosine']:
        kde = KernelDensity(bandwidth, kernel=kernel).fit(X)
        assert_raises(NotImplementedError, kde.sample, 100)

    # non-regression test: used to return a scalar
    X = rng.randn(4, 1)
    kde = KernelDensity(kernel="gaussian").fit(X)
    assert_equal(kde.sample().shape, (1, 1))

def test_kde_algorithm_metric_choice():
    """Smoke test for various metrics and algorithms"""
    rng = np.random.RandomState(0)
    X = rng.randn(10, 2)    # 2 features required for haversine dist.
    Y = rng.randn(10, 2)

    for algorithm in ['auto', 'ball_tree', 'kd_tree']:
        for metric in ['euclidean', 'minkowski', 'manhattan',
                       'chebyshev', 'haversine']:
            if algorithm == 'kd_tree' and metric not in KDTree.valid_metrics:
                assert_raises(ValueError, KernelDensity,
                              algorithm=algorithm, metric=metric)
            else:
                kde = KernelDensity(algorithm=algorithm, metric=metric)
                kde.fit(X)
                y_dens = kde.score_samples(Y)
                assert_equal(y_dens.shape, Y.shape[:1])

def test_kde_score(n_samples=100, n_features=3):
    pass
    #FIXME
    #np.random.seed(0)
    #X = np.random.random((n_samples, n_features))
    #Y = np.random.random((n_samples, n_features))

def test_kde_badargs():
    assert_raises(ValueError, KernelDensity,
                  algorithm='blah')
    assert_raises(ValueError, KernelDensity,
                  bandwidth=0)
    assert_raises(ValueError, KernelDensity,
                  kernel='blah')
    assert_raises(ValueError, KernelDensity,
                  metric='blah')
    assert_raises(ValueError, KernelDensity,
                  algorithm='kd_tree', metric='blah')

def test_kde_pipeline_gridsearch():
    # test that kde plays nice in pipelines and grid-searches
    X, _ = make_blobs(cluster_std=.1, random_state=1,
                      centers=[[0, 1], [1, 0], [0, 0]])
    pipe1 = make_pipeline(StandardScaler(with_mean=False, with_std=False),
                          KernelDensity(kernel="gaussian"))
    params = dict(kerneldensity__bandwidth=[0.001, 0.01, 0.1, 1, 10])
    search = GridSearchCV(pipe1, param_grid=params, cv=5)
    search.fit(X)
    assert_equal(search.best_params_['kerneldensity__bandwidth'], .1)

if __name__ == '__main__':
    import nose
    nose.runmodule()


"""Module for performing plotting for ``mutpath`` package.

This module uses ``pylab`` and ``matplotlib`` to make plots. These plots will
fail if ``pylab`` and ``matplotlib`` are not available for importation. Before
running any function in this module, you can run the *PylabAvailable*
function to determine if ``pylab`` and ``matplotlib`` are available. Otherwise,
calling any other function will raise an Exception if thise modules are
not available. The ``pdf`` backend is used for ``matplotlib`` / ``pylab``. This means
that plots must be created as PDF files.

Functions are:

`PylabAvailable`

`CumulativeFractionPlot`

'DatesPlot`

`Base10Formatter`

`SplitLabel`

Written by Jesse Bloom.
"""

import os
import sys
import math

# global variable _pylabavailable indicates if pylab/matplotlib present
try:
    import matplotlib
    matplotlib.use('pdf')
    import pylab
    _pylabavailable = True
except ImportError:
    _pylabavailable = False

def PylabAvailable():
    """Returns True if pylab/matplotlib available, False otherwise.

    You should call this function to test for the availability of the
    pylab/matplotlib plotting modules before using other functions in
    this module.
    """
    return _pylabavailable

def DatesPlot(mutdates, plotfile, interval):
    """Plots dates of mutations.

    Uses pylab / matplotlib to plot the dates and credible intervals
    for mutations. Will raise an error *PylabAvailable() == False*.
    The plot is a PDF.

    * *mutdates* is a list of the mutations, in the form of the tuples
      *(median, mininterval, maxinterval, mut, fractoca, weight)*. Mutations
      are plotted in the order they are listed. In these tuples:

      * *median* : posterior median date

      * *minterval* : minimum of credible interval

      * *maxinterval* : maximum of credible interval

      * *mut* : string giving name of mutation

      * *fractoca* : probability mutation is on path from common ancestor
        to starting sequence

      * *weight* : fraction of paths containing mutation.

    * *plotfile* is a string giving the name of the PDF file we create.

    * *interval* is the range of the credible interval. For example, 0.9
      means a 90% credible interval.
    """
    ext = os.path.splitext(plotfile)[1].lower()
    if ext != '.pdf':
        raise ValueError("Extension must be .pdf, but found %s" % ext)
    if not PylabAvailable():
        raise ValueError("pylab / matplotlib not available.")
    if not mutdates:
        raise ValueError("no mutation dates to plot")
    tocalabels = []
    tocamedians = []
    tocaerrlow = []
    tocaerrhigh = []
    tocays = []
    fromcalabels = []
    fromcamedians = []
    fromcaerrlow = []
    fromcaerrhigh = []
    fromcays = []
    y = 0
    for (median, mininterval, maxinterval, mut, fractoca, weight) in mutdates:
        label = "%s" % (mut)
        errlow = median - mininterval
        errhigh = maxinterval - median
        if fractoca > 0.5:
            tocays.append(y)
            tocalabels.append(label)
            tocamedians.append(median)
            tocaerrlow.append(errlow)
            tocaerrhigh.append(errhigh)
        else:
            fromcays.append(y)
            fromcalabels.append(label)
            fromcamedians.append(median)
            fromcaerrlow.append(errlow)
            fromcaerrhigh.append(errhigh)
        y += 1
    (lmargin, rmargin, bmargin, tmargin) = (0.11, 0.05, 0.08, 0.01)
    matplotlib.rc('font', size=10)
    matplotlib.rc('xtick', labelsize=10)
    matplotlib.rc('ytick', labelsize=10)
    matplotlib.rc('legend', numpoints=1)
    matplotlib.rc('legend', fontsize=10)
    fig = pylab.figure(figsize=(6, 6))
    ax = pylab.axes([lmargin, bmargin, 1 - lmargin - rmargin, 1 - tmargin - bmargin])
    tocabar = fromcabar = None
    if tocalabels:
        tocabar = pylab.errorbar(tocamedians, tocays, xerr=[tocaerrlow, tocaerrhigh], fmt='sr')
    if fromcalabels:
        fromcabar = pylab.errorbar(fromcamedians, fromcays, xerr=[fromcaerrlow, fromcaerrhigh], fmt='sb')
    ny = len(mutdates)
    pylab.gca().set_ylim((-1, ny))
    pylab.gca().yaxis.set_major_locator(matplotlib.ticker.FixedLocator([y for y in range(ny)]))
    pylab.gca().yaxis.set_major_formatter(matplotlib.ticker.FixedFormatter(tocalabels + fromcalabels))
    pylab.gca().xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter(useOffset=False))
    pylab.xlabel("Date (posterior median and Bayesian %.2f%s credible interval)" % (interval * 100, '%'))
    if tocabar and fromcabar:
        pylab.legend([tocabar[0], fromcabar[0]], ['path to common ancestor', 'path from common ancestor'], loc='lower right')
    elif tocabar:
        pylab.legend([tocabar[0]], ['path to common ancestor'], loc='lower right')
    elif fromcabar:
        pylab.legend([fromcabar[0]], ['path from common ancestor'], loc='lower right')
    pylab.savefig(plotfile)

def CumulativeFractionPlot(datalist, plotfile, title, xlabel):
    """Creates a cumulative fraction plot.

    Takes a list of numeric data. Plots a cumulative fraction
    plot giving the fraction of the data points that are <=
    the indicated value.

    *datalist* is a list of numbers giving the data for which we
    are computing the cumulative fraction plot. Raises an
    exception if this is an empty list.

    *plotfile* is the name of the output plot file created by this method
    (such as 'plot.pdf'). The extension must be '.pdf'.

    *title* is a string placed above the plot as a title. Uses LaTex
    formatting.

    *xlabel* is the label given to the X-axis. Uses LaTex formatting.

    This function uses pylab / matplotlib. It will raise an Exception if
    these modules cannot be imported (if PylabAvailable() is False).
    """
    if len(datalist) < 1:
        raise ValueError("datalist is empty")
    if not _pylabavailable:
        raise ImportError("Could not find pylab or matplotlib")
    if os.path.splitext(plotfile)[1] != '.pdf':
        raise ValueError("plotfile must end in .pdf: %s" % plotfile)
    datalist.sort() # sort from smallest to largest
    (xmin, xmax) = (datalist[0], datalist[-1])
    n = len(datalist)
    cumfracs = []
    cf = 0.0
    for x in datalist:
        cf += 1. / n
        cumfracs.append(cf)
    assert len(datalist) == len(cumfracs)
    assert abs(1.0 - cf) < 1e-7
    matplotlib.rc('text', usetex=True)
    matplotlib.rc('font', size=12)
    fig = pylab.figure(figsize=(6, 4))
    (lmargin, rmargin, bmargin, tmargin) = (0.1, 0.01, 0.15, 0.1)
    ax = pylab.axes([lmargin, bmargin, 1 - lmargin - rmargin, 1 -\
            bmargin - tmargin])
    pylab.plot(datalist, cumfracs, 'r-')
    pylab.gca().set_ylim([0, 1])
    pylab.gca().set_xlim([xmin, xmax])
    pylab.ylabel('cumulative fraction')
    pylab.xlabel(xlabel)
    pylab.title(title)
    if plotfile:
        pylab.savefig(plotfile)
    pylab.clf()
    pylab.close()

def Base10Formatter(number, exp_cutoff, exp_decimal_digits, decimal_digits):
    """Converts a number into Latex formatting with scientific notation.

    Takes a number and converts it to a string that can be shown
    in LaTex using math mode. It is converted to scientific notation
    if the criteria specified by exp_cutoff.

    *number* the number to be formatted, should be a float or integer.
    Currently only works for numbers >= 0

    *exp_cutoff* convert to scientific notation if abs(math.log10(number)) >= this.

    *exp_decimal_digits* show this many digits after the decimal if number
    is converted to scientific notation.

    *decimal_digits* show this many digits after the decimal if number
    is NOT converted to scientific notation.

    The returned value is the LaTex' string. If the number is zero, the
    returned string is simply '0'.

    >>> Base10Formatter(103, 3, 1, 1)
    '103.0'

    >>> Base10Formatter(103.0, 2, 1, 1)
    '1.0 \\\\times 10^{2}'

    >>> Base10Formatter(103.0, 2, 2, 1)
    '1.03 \\\\times 10^{2}'

    >>> Base10Formatter(2892.3, 3, 1, 1)
    '2.9 \\\\times 10^{3}'

    >>> Base10Formatter(0.0, 3, 1, 1)
    '0'

    >>> Base10Formatter(0.012, 2, 1, 1)
    '1.2 \\\\times 10^{-2}'

    >>> Base10Formatter(-0.1, 3, 1, 1)
    Traceback (most recent call last):
        ...
    ValueError: number must be >= 0
    """
    if number < 0:
        raise ValueError('number must be >= 0')
    if number == 0:
        return '0'
    exponent = int(math.log10(number))
    if math.log10(number) < exponent and number < 1:
        exponent -= 1
    if abs(exponent) >= exp_cutoff:
        x = number / (10.**exponent)
        formatstr = '%.' + '%d' % exp_decimal_digits + 'f \\times 10^{%d}'
        return formatstr % (x, exponent)
    else:
        formatstr = '%.' + '%d' % decimal_digits + 'f'
        return formatstr % number

def SplitLabel(label, splitlen, splitchar):
    """Splits a string with a return if it exceeds a certain length.

    *label* a string giving the label we might split.

    *splitlen* the maximum length of a label before we attempt to
    split it.

    *splitchar* the character added when splitting a label.

    If len(*label*) > *splitlen*, we attempt to split the label in the
    middle by adding *splitchar*. The label is split as close to the
    middle as possible while splitting at a space.

    No splitting as label length less than *splitlen*

    >>> SplitLabel('WT virus 1', 10, '\\n')
    'WT virus 1'

    Splitting of this label

    >>> SplitLabel('WT plasmid 1', 10, '\\n')
    'WT\\nplasmid 1'

    Splitting of this label

    >>> SplitLabel('mutated WT plasmid 1', 10, '\\n')
    'mutated WT\\nplasmid 1'

    """
    if len(label) <= splitlen:
        return label
    else:
        j = 0
        imid = len(label) // 2
        index = None
        while 0 <= imid - j <= imid + j < len(label):
            if label[imid - j].isspace():
                return "%s%s%s" % (label[ : imid - j], splitchar, label[imid - j + 1 : ])
            elif label[imid + j].isspace():
                return "%s%s%s" % (label[ : imid + j], splitchar, label[imid + j + 1 : ])
            j += 1
        else:
            return label # no white space to split

if __name__ == '__main__':
    import doctest
    doctest.testmod()


"""
Agglomerative clustering with and without structure
===================================================

This example shows the effect of imposing a connectivity graph to capture
local structure in the data. The graph is simply the graph of 20 nearest
neighbors.

Two consequences of imposing a connectivity can be seen. First clustering
with a connectivity matrix is much faster.

Second, when using a connectivity matrix, average and complete linkage are
unstable and tend to create a few clusters that grow very quickly. Indeed,
average and complete linkage fight this percolation behavior by considering all
the distances between two clusters when merging them. The connectivity
graph breaks this mechanism. This effect is more pronounced for very
sparse graphs (try decreasing the number of neighbors in
kneighbors_graph) and with complete linkage. In particular, having a very
small number of neighbors in the graph, imposes a geometry that is
close to that of single linkage, which is well known to have this
percolation instability.
"""
# Authors: Gael Varoquaux, Nelle Varoquaux
# License: BSD 3 clause

import time
import matplotlib.pyplot as plt
import numpy as np

from sklearn.cluster import AgglomerativeClustering
from sklearn.neighbors import kneighbors_graph

# Generate sample data
n_samples = 1500
np.random.seed(0)
t = 1.5 * np.pi * (1 + 3 * np.random.rand(1, n_samples))
x = t * np.cos(t)
y = t * np.sin(t)

X = np.concatenate((x, y))
X += .7 * np.random.randn(2, n_samples)
X = X.T

# Create a graph capturing local connectivity. Larger number of neighbors
# will give more homogeneous clusters to the cost of computation
# time. A very large number of neighbors gives more evenly distributed
# cluster sizes, but may not impose the local manifold structure of
# the data
knn_graph = kneighbors_graph(X, 30, include_self=False)

for connectivity in (None, knn_graph):
    for n_clusters in (30, 3):
        plt.figure(figsize=(10, 4))
        for index, linkage in enumerate(('average', 'complete', 'ward')):
            plt.subplot(1, 3, index + 1)
            model = AgglomerativeClustering(linkage=linkage,
                                            connectivity=connectivity,
                                            n_clusters=n_clusters)
            t0 = time.time()
            model.fit(X)
            elapsed_time = time.time() - t0
            plt.scatter(X[:, 0], X[:, 1], c=model.labels_,
                        cmap=plt.cm.spectral)
            plt.title('linkage=%s (time %.2fs)' % (linkage, elapsed_time),
                      fontdict=dict(verticalalignment='top'))
            plt.axis('equal')
            plt.axis('off')

            plt.subplots_adjust(bottom=0, top=.89, wspace=0,
                                left=0, right=1)
            plt.suptitle('n_cluster=%i, connectivity=%r' %
                         (n_clusters, connectivity is not None), size=17)

plt.show()