import json as _json_io
import textwrap
import typing
from datetime import timedelta

import cauldron as _cd
from cauldron import environ
from cauldron import render
from cauldron.render import plots as render_plots
from cauldron.render import texts as render_texts
from cauldron.session import report

def _get_report() -> 'report.Report':
    """Fetches the report associated with the currently running step."""
    return _cd.project.get_internal_project().current_step.report

def inspect(source: dict):
    """
    Inspects the data and structure of the source dictionary object and
    adds the results to the display for viewing.

    :param source:
        A dictionary object to be inspected.
    :return:
    """
    r = _get_report()
    r.append_body(render.inspect(source))

def header(header_text: str, level: int = 1, expand_full: bool = False):
    """
    Adds a text header to the display with the specified level.

    :param header_text:
        The text to display in the header.
    :param level:
        The level of the header, which corresponds to the html header
        levels, such as <h1>, <h2>, ...
    :param expand_full:
        Whether or not the header will expand to fill the width of the entire
        notebook page, or be constrained by automatic maximum page width. The
        default value of False lines the header up with text displays.
    """
    r = _get_report()
    r.append_body(render.header(
        header_text,
        level=level,
        expand_full=expand_full
    ))

def text(value: str, preformatted: bool = False):
    """
    Adds text to the display. If the text is not preformatted, it will be
    displayed in paragraph format. Preformatted text will be displayed
    inside a pre tag with a monospace font.

    :param value:
        The text to display.
    :param preformatted:
        Whether or not to preserve the whitespace display of the text.
    """
    if preformatted:
        result = render_texts.preformatted_text(value)
    else:
        result = render_texts.text(value)
    r = _get_report()
    r.append_body(result)
    r.stdout_interceptor.write_source(
        '{}\n'.format(textwrap.dedent(value))
    )

def markdown(
        source: str = None,
        source_path: str = None,
        preserve_lines: bool = False,
        font_size: float = None,
        **kwargs
):
    """
    Renders the specified source string or source file using markdown and
    adds the resulting HTML to the notebook display.

    :param source:
        A markdown formatted string.
    :param source_path:
        A file containing markdown text.
    :param preserve_lines:
        If True, all line breaks will be treated as hard breaks. Use this
        for pre-formatted markdown text where newlines should be retained
        during rendering.
    :param font_size:
        Specifies a relative font size adjustment. The default value is 1.0,
        which preserves the inherited font size values. Set it to a value
        below 1.0 for smaller font-size rendering and greater than 1.0 for
        larger font size rendering.
    :param kwargs:
        Any variable replacements to make within the string using Jinja2
        templating syntax.
    """
    r = _get_report()

    result = render_texts.markdown(
        source=source,
        source_path=source_path,
        preserve_lines=preserve_lines,
        font_size=font_size,
        **kwargs
    )
    r.library_includes += result['library_includes']

    r.append_body(result['body'])
    r.stdout_interceptor.write_source(
        '{}\n'.format(textwrap.dedent(result['rendered']))
    )

def json(**kwargs):
    """
    Adds the specified data to the the output display window with the
    specified key. This allows the user to make available arbitrary
    JSON-compatible data to the display for runtime use.

    :param kwargs:
        Each keyword argument is added to the CD.data object with the
        specified key and value.
    """
    r = _get_report()
    r.append_body(render.json(**kwargs))
    r.stdout_interceptor.write_source(
        '{}\n'.format(_json_io.dumps(kwargs, indent=2))
    )

def plotly(
        data: typing.Union[dict, list, typing.Any] = None,
        layout: typing.Union[dict, typing.Any] = None,
        scale: float = 0.5,
        figure: typing.Union[dict, typing.Any] = None,
        static: bool = False
):
    """
    Creates a Plotly plot in the display with the specified data and
    layout.

    :param data:
        The Plotly trace data to be plotted.
    :param layout:
        The layout data used for the plot.
    :param scale:
        The display scale with units of fractional screen height. A value
        of 0.5 constrains the output to a maximum height equal to half the
        height of browser window when viewed. Values below 1.0 are usually
        recommended so the entire output can be viewed without scrolling.
    :param figure:
        In cases where you need to create a figure instead of separate data
        and layout information, you can pass the figure here and leave the
        data and layout values as None.
    :param static:
        If true, the plot will be created without interactivity.
        This is useful if you have a lot of plots in your notebook.
    """
    r = _get_report()

    if not figure and not isinstance(data, (list, tuple)):
        data = [data]

    if 'plotly' not in r.library_includes:
        r.library_includes.append('plotly')

    r.append_body(render.plotly(
        data=data,
        layout=layout,
        scale=scale,
        figure=figure,
        static=static
    ))
    r.stdout_interceptor.write_source('[ADDED] Plotly plot\n')

def table(
        data_frame,
        scale: float = 0.7,
        include_index: bool = False,
        max_rows: int = 500,
        sample_rows: typing.Optional[int] = None,
        formats: typing.Union[
            str,
            typing.Callable[[typing.Any], str],
            typing.Dict[
                str,
                typing.Union[str, typing.Callable[[typing.Any], str]]
            ]
        ] = None
):
    """
    Adds the specified data frame to the display in a nicely formatted
    scrolling table.

    :param data_frame:
        The pandas data frame to be rendered to a table.
    :param scale:
        The display scale with units of fractional screen height. A value
        of 0.5 constrains the output to a maximum height equal to half the
        height of browser window when viewed. Values below 1.0 are usually
        recommended so the entire output can be viewed without scrolling.
    :param include_index:
        Whether or not the index column should be included in the displayed
        output. The index column is not included by default because it is
        often unnecessary extra information in the display of the data.
    :param max_rows:
        This argument exists to prevent accidentally writing very large data
        frames to a table, which can cause the notebook display to become
        sluggish or unresponsive. If you want to display large tables, you need
        only increase the value of this argument.
    :param sample_rows:
        When set to a positive integer value, the DataFrame will be randomly
        sampled to the specified number of rows when displayed in the table.
        If the value here is larger than the number of rows in the DataFrame,
        the sampling will have no effect and the entire DataFrame will be
        displayed instead.
    :param formats:
        An optional dictionary that, when specified, should contain a mapping
        between column names and formatting strings to apply to that column
        for display purposes. For example, ``{'foo': '{:,.2f}%'}`` would
        transform a column ``foo = [12.2121, 34.987123, 42.72839]`` to
        display as ``foo = [12.21%, 34.99%, 42.73%]``. The formatters should
        follow the standard Python string formatting guidelines the same as
        the ``str.format()`` command having the value of the column as the only
        positional argument in the format arguments. A string value can also
        be specified for uniform formatting of all columns (or if displaying
        a series with only a single value).
    """
    r = _get_report()
    r.append_body(render.table(
        data_frame=data_frame,
        scale=scale,
        include_index=include_index,
        max_rows=max_rows,
        sample_rows=sample_rows,
        formats=formats
    ))
    r.stdout_interceptor.write_source('[ADDED] Table\n')

def svg(svg_dom: str, filename: str = None):
    """
    Adds the specified SVG string to the display. If a filename is
    included, the SVG data will also be saved to that filename within the
    project results folder.

    :param svg_dom:
        The SVG string data to add to the display.
    :param filename:
        An optional filename where the SVG data should be saved within
        the project results folder.
    """
    r = _get_report()
    r.append_body(render.svg(svg_dom))
    r.stdout_interceptor.write_source('[ADDED] SVG\n')

    if not filename:
        return

    if not filename.endswith('.svg'):
        filename += '.svg'

    r.files[filename] = svg_dom

def jinja(path: str, **kwargs):
    """
    Renders the specified Jinja2 template to HTML and adds the output to the
    display.

    :param path:
        The fully-qualified path to the template to be rendered.
    :param kwargs:
        Any keyword arguments that will be use as variable replacements within
        the template.
    """
    r = _get_report()
    r.append_body(render.jinja(path, **kwargs))
    r.stdout_interceptor.write_source('[ADDED] Jinja2 rendered HTML\n')

def whitespace(lines: float = 1.0):
    """
    Adds the specified number of lines of whitespace.

    :param lines:
        The number of lines of whitespace to show.
    """
    r = _get_report()
    r.append_body(render.whitespace(lines))
    r.stdout_interceptor.write_source('\n')

def image(
        filename: str,
        width: int = None,
        height: int = None,
        justify: str = 'left'
):
    """
    Adds an image to the display. The image must be located within the
    assets directory of the Cauldron notebook's folder.

    :param filename:
        Name of the file within the assets directory,
    :param width:
        Optional width in pixels for the image.
    :param height:
        Optional height in pixels for the image.
    :param justify:
        One of 'left', 'center' or 'right', which specifies how the image
        is horizontally justified within the notebook display.
    """
    r = _get_report()
    path = '/'.join(['reports', r.project.uuid, 'latest', 'assets', filename])
    r.append_body(render.image(path, width, height, justify))
    r.stdout_interceptor.write_source('[ADDED] Image\n')

def html(dom: str):
    """
    A string containing a valid HTML snippet.

    :param dom:
        The HTML string to add to the display.
    """
    r = _get_report()
    r.append_body(render.html(dom))
    r.stdout_interceptor.write_source('[ADDED] HTML\n')

def workspace(show_values: bool = True, show_types: bool = True):
    """
    Adds a list of the shared variables currently stored in the project
    workspace.

    :param show_values:
        When true the values for each variable will be shown in addition to
        their name.
    :param show_types:
        When true the data types for each shared variable will be shown in
        addition to their name.
    """
    r = _get_report()

    data = {}
    for key, value in r.project.shared.fetch(None).items():
        if key.startswith('__cauldron_'):
            continue
        data[key] = value

    r.append_body(render.status(data, values=show_values, types=show_types))

def pyplot(
        figure=None,
        scale: float = 0.8,
        clear: bool = True,
        aspect_ratio: typing.Union[list, tuple] = None
):
    """
    Creates a matplotlib plot in the display for the specified figure. The size
    of the plot is determined automatically to best fit the notebook.

    :param figure:
        The matplotlib figure to plot. If omitted, the currently active
        figure will be used.
    :param scale:
        The display scale with units of fractional screen height. A value
        of 0.5 constrains the output to a maximum height equal to half the
        height of browser window when viewed. Values below 1.0 are usually
        recommended so the entire output can be viewed without scrolling.
    :param clear:
        Clears the figure after it has been rendered. This is useful to
        prevent persisting old plot data between repeated runs of the
        project files. This can be disabled if the plot is going to be
        used later in the project files.
    :param aspect_ratio:
        The aspect ratio for the displayed plot as a two-element list or
        tuple. The first element is the width and the second element the
        height. The units are "inches," which is an important consideration
        for the display of text within the figure. If no aspect ratio is
        specified, the currently assigned values to the plot will be used
        instead.
    """
    r = _get_report()
    r.append_body(render_plots.pyplot(
        figure,
        scale=scale,
        clear=clear,
        aspect_ratio=aspect_ratio
    ))
    r.stdout_interceptor.write_source('[ADDED] PyPlot plot\n')

def bokeh(model, scale: float = 0.7, responsive: bool = True):
    """
    Adds a Bokeh plot object to the notebook display.

    :param model:
        The plot object to be added to the notebook display.
    :param scale:
        How tall the plot should be in the notebook as a fraction of screen
        height. A number between 0.1 and 1.0. The default value is 0.7.
    :param responsive:
        Whether or not the plot should responsively scale to fill the width
        of the notebook. The default is True.
    """
    r = _get_report()

    if 'bokeh' not in r.library_includes:
        r.library_includes.append('bokeh')

    r.append_body(render_plots.bokeh_plot(
        model=model,
        scale=scale,
        responsive=responsive
    ))
    r.stdout_interceptor.write_source('[ADDED] Bokeh plot\n')

def listing(
        source: list,
        ordered: bool = False,
        expand_full: bool = False
):
    """
    An unordered or ordered list of the specified *source* iterable where
    each element is converted to a string representation for display.

    :param source:
        The iterable to display as a list.
    :param ordered:
        Whether or not the list should be ordered. If False, which is the
        default, an unordered bulleted list is created.
    :param expand_full:
        Whether or not the list should expand to fill the screen horizontally.
        When defaulted to False, the list is constrained to the center view
        area of the screen along with other text. This can be useful to keep
        lists aligned with the text flow.
    """
    r = _get_report()
    r.append_body(render.listing(
        source=source,
        ordered=ordered,
        expand_full=expand_full
    ))
    r.stdout_interceptor.write_source('[ADDED] Listing\n')

def list_grid(
        source: list,
        expand_full: bool = False,
        column_count: int = 2,
        row_spacing: float = 1.0
):
    """
    An multi-column list of the specified *source* iterable where
    each element is converted to a string representation for display.

    :param source:
        The iterable to display as a list.
    :param expand_full:
        Whether or not the list should expand to fill the screen horizontally.
        When defaulted to False, the list is constrained to the center view
        area of the screen along with other text. This can be useful to keep
        lists aligned with the text flow.
    :param column_count:
        The number of columns to display. The specified count is applicable to
        high-definition screens. For Lower definition screens the actual count
        displayed may be fewer as the layout responds to less available
        horizontal screen space.
    :param row_spacing:
        The number of lines of whitespace to include between each row in the
        grid. Set this to 0 for tightly displayed lists.
    """
    r = _get_report()
    r.append_body(render.list_grid(
        source=source,
        expand_full=expand_full,
        column_count=column_count,
        row_spacing=row_spacing
    ))
    r.stdout_interceptor.write_source('[ADDED] List grid\n')

def latex(source: str):
    """
    Add a mathematical equation in latex math-mode syntax to the display.
    Instead of the traditional backslash escape character, the @ character is
    used instead to prevent backslash conflicts with Python strings. For
    example, \\delta would be @delta.

    :param source:
        The string representing the latex equation to be rendered.
    """
    r = _get_report()
    if 'katex' not in r.library_includes:
        r.library_includes.append('katex')

    r.append_body(render_texts.latex(source.replace('@', '\\')))
    r.stdout_interceptor.write_source('[ADDED] Latex equation\n')

def head(source, count: int = 5):
    """
    Displays a specified number of elements in a source object of many
    different possible types.

    :param source:
        DataFrames will show *count* rows of that DataFrame. A list, tuple or
        other iterable, will show the first *count* rows. Dictionaries will
        show *count* keys from the dictionary, which will be randomly selected
        unless you are using an OrderedDict. Strings will show the first
        *count* characters.
    :param count:
        The number of elements to show from the source.
    """
    r = _get_report()
    r.append_body(render_texts.head(source, count=count))
    r.stdout_interceptor.write_source('[ADDED] Head\n')

def tail(source, count: int = 5):
    """
    The opposite of the head function. Displays the last *count* elements of
    the *source* object.

    :param source:
        DataFrames will show the last *count* rows of that DataFrame. A list,
        tuple or other iterable, will show the last *count* rows. Dictionaries
        will show *count* keys from the dictionary, which will be randomly
        selected unless you are using an OrderedDict. Strings will show the
        last *count* characters.
    :param count:
        The number of elements to show from the source.
    """
    r = _get_report()
    r.append_body(render_texts.tail(source, count=count))
    r.stdout_interceptor.write_source('[ADDED] Tail\n')

def status(
        message: str = None,
        progress: float = None,
        section_message: str = None,
        section_progress: float = None,
):
    """
    Updates the status display, which is only visible while a step is running.
    This is useful for providing feedback and information during long-running
    steps.

    A section progress is also available for cases where long running tasks
    consist of multiple tasks and you want to display sub-progress messages
    within the context of the larger status.

    Note: this is only supported when running in the Cauldron desktop
    application.

    :param message:
        The status message you want to display. If left blank the previously
        set status message will be retained. Should you desire to remove an
        existing message, specify a blank string for this argument.
    :param progress:
        A number between zero and one that indicates the overall progress for
        the current status. If no value is specified, the previously assigned
        progress will be retained.
    :param section_message:
        The status message you want to display for a particular task within a
        long-running step. If left blank the previously set section message
        will be retained. Should you desire to remove an existing message,
        specify a blank string for this argument.
    :param section_progress:
        A number between zero and one that indicates the progress for the
        current section status. If no value is specified, the previously
        assigned section progress value will be retained.
    """
    environ.abort_thread()
    r = _get_report()
    step = _cd.project.get_internal_project().current_step

    changes = 0
    has_changed = step.progress_message != message
    if message is not None and has_changed:
        changes += 1
        step.progress_message = message

    has_changed = step.progress_message != max(0, min(1, progress or 0))
    if progress is not None and has_changed:
        changes += 1
        step.progress = max(0.0, min(1.0, progress))

    has_changed = step.sub_progress_message != section_message
    if section_message is not None and has_changed:
        changes += 1
        step.sub_progress_message = section_message

    has_changed = step.sub_progress != max(0, min(1, section_progress or 0))
    if section_progress is not None and has_changed:
        changes += 1
        step.sub_progress = section_progress

    if changes > 0:
        # update the timestamp to inform rendering that a status
        # has changed and should be re-rendered into the step.
        r.update_last_modified()

def code_block(
        code: str = None,
        path: str = None,
        language_id: str = None,
        title: str = None,
        caption: str = None
):
    """
    Adds a block of syntax highlighted code to the display from either
    the supplied code argument, or from the code file specified
    by the path argument.

    :param code:
        A string containing the code to be added to the display
    :param path:
        A path to a file containing code to be added to the display
    :param language_id:
        The language identifier that indicates what language should
        be used by the syntax highlighter. Valid values are any of the
        languages supported by the Pygments highlighter.
    :param title:
        If specified, the code block will include a title bar with the
        value of this argument
    :param caption:
        If specified, the code block will include a caption box below the code
        that contains the value of this argument
    """
    environ.abort_thread()
    r = _get_report()
    r.append_body(render.code_block(
        block=code,
        path=path,
        language=language_id,
        title=title,
        caption=caption
    ))
    r.stdout_interceptor.write_source('{}\n'.format(code))

def elapsed():
    """
    Displays the elapsed time since the step started running.
    """
    environ.abort_thread()
    step = _cd.project.get_internal_project().current_step
    r = _get_report()
    r.append_body(render.elapsed_time(step.elapsed_time))

    result = '[ELAPSED]: {}\n'.format(timedelta(seconds=step.elapsed_time))
    r.stdout_interceptor.write_source(result)


# =============================================================================
# Authors: PAR Government
# Organization: DARPA
#
# Copyright (c) 2016 PAR Government
# All rights reserved.
#
#
# adapted from https://github.com/enmasse/jpeg_read
#==============================================================================

import sys
from math import *
from Tkinter import *
import matplotlib.pyplot as plt
import numpy as np
import logging
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg

def memoize (function):
    # http://programmingzen.com/2009/05/18/memoization-in-ruby-and-python/
    cache = {}

    def decorated_function (*args):
        try:
            return cache[args]
        except KeyError:
            val = function (*args)
            cache[args] = val
            return val

    return decorated_function

@memoize
def decodeBits (len, val):
    """ Calculate the value from the "additional" bits in the huffman data. """

    return val if (val & (1 << len - 1)) else val - ((1 << len) - 1)

def extractCoeffs (data):
    dclum = []
    dcchr1 = []
    dcchr2 = []
    aclum = []
    acchr1 = []
    acchr2 = []
    for MCU in data:
        lum = MCU[0]
        chr1 = MCU[1]
        chr2 = MCU[2]
        for MCU_component in lum:
            if len (MCU_component):
                dclum.append (MCU_component[0])
                aclum.extend (MCU_component[1:])
        for MCU_component in chr1:
            if len (MCU_component):
                dcchr1.append (MCU_component[0])
                acchr1.extend (MCU_component[1:])
        for MCU_component in chr2:
            if len (MCU_component):
                dcchr2.append (MCU_component[0])
                acchr2.extend (MCU_component[1:])

    return (dclum, dcchr1, dcchr2, aclum, acchr1, acchr2)

def generateHuffmanCodes (huffsize):
    """ Calculate the huffman code of each length. """
    huffcode = []
    k = 0
    code = 0

    # Magic
    for i in range (len (huffsize)):
        si = huffsize[i]
        for k in range (si):
            huffcode.append ((i + 1, code))
            code += 1

        code <<= 1

    return huffcode

def getBits (num, gen):
    """ Get "num" bits from gen. """
    out = 0
    for i in range (num):
        out <<= 1
        val = gen.next ()
        if val != []:
            out += val & 0x01
        else:
            return []

    return out

def mapHuffmanCodes (codes, values):
    """ Map the huffman code to the right value. """
    out = {}

    for i in range (len (codes)):
        out[codes[i]] = values[i]

    return out

def readAPP (type, file):
    """ Read APP marker. """
    Lp = readWord (file)
    Lp -= 2

    # If APP0 try to read the JFIF header
    # Not really necessary
    if type == 0:
        identifier = file.read (5)
        Lp -= 5
        version = file.read (2)
        Lp -= 2
        units = ord (file.read (1))
        Lp -= 1
        Xdensity = ord (file.read (1)) << 8
        Xdensity |= ord (file.read (1))
        Lp -= 2
        Ydensity = ord (file.read (1)) << 8
        Ydensity |= ord (file.read (1))
        Lp -= 2

    file.seek (Lp, 1)

def readByte (file):
    """ Read a byte from file. """
    return ord (file.read (1))

def readWord (file):
    """ Read a 16 bit word from file. """
    return ord (file.read (1)) << 8 | ord (file.read (1))

def restoreDC (data):
    """ Restore the DC values.  They are coded as the difference from the
        previous DC value of the same component.
    """

    out = []
    dc_prev = [0 for x in range (len (data[0]))]

    # For each MCU
    for mcu in data:
        # For each component
        for comp_num in range (len (mcu)):
            # For each DU
            for du in range (len (mcu[comp_num])):
                if mcu[comp_num][du]:
                    mcu[comp_num][du][0] += dc_prev[comp_num]
                    dc_prev[comp_num] = mcu[comp_num][du][0]

        out.append (mcu)

    return out

class JPEG_Reader:
    """ Class for reading DCT coefficients from JPEG files. """

    def __init__ (self):
        self.huffman_ac_tables = [{}, {}, {}, {}]
        self.huffman_dc_tables = [{}, {}, {}, {}]
        self.q_table = [[], [], [], []]

        self.XYP = 0, 0, 0
        self.component = {}
        self.num_components = 0
        self.mcus_read = 0
        self.dc = []
        self.inline_dc = 0
        self.bit_stream = []
        self.EOI = False

    def readDCT_Coeffs (self, filename):
        """ Reads and returns DCT coefficients from the supplied JPEG file. """

        self.__init__ ()
	data = []

        with open (filename, "rb") as inputFile:
            in_char = inputFile.read (1)
            while in_char:
                if in_char == chr (0xff):
                    in_char = inputFile.read (1)
                    in_num = ord (in_char)
                    if 0xe0 <= in_num <= 0xef:
                        readAPP (in_num - 0xe0, inputFile)
                    elif in_num == 0xdb:
                        self.__readDQT (inputFile)
                    elif in_num == 0xdc:
                        self.__readDNL (inputFile)
                    elif in_num == 0xc4:
                        self.__readDHT (inputFile)
                    elif in_num == 0xc8:
                        print "JPG"
                    elif 0xc0 <= in_num <= 0xcf:
                        self.__readSOF (in_num - 0xc0, inputFile)
                    elif in_num == 0xda:
                        self.__readSOS (inputFile)
                        self.bit_stream = self.__readBit (inputFile)
                        while not self.EOI:
                            data.append (self.__readMCU ())
                in_char = inputFile.read (1)

        return extractCoeffs (data if self.inline_dc else restoreDC (data))

    def __readBit (self, file):
        """ A generator that reads one bit from file and handles markers and
            byte stuffing.
        """

        input = file.read (1)
        while input and not self.EOI:
            if input == chr (0xFF):
                cmd = file.read (1)
                if cmd:
                    # Byte stuffing
                    if cmd == chr (0x00):
                        input = chr (0xFF)
                    # End of image marker
                    elif cmd == chr (0xD9):
                        self.EOI = True
                    # Restart markers
                    elif 0xD0 <= ord (cmd) <= 0xD7 and self.inline_dc:
                        # Reset dc value
                        self.dc = [0 for i in range (self.num_components + 1)]
                        input = file.read (1)
                    else:
                        input = file.read (1)
                        #print "CMD: %x" % ord(cmd)

            if not self.EOI:
                for i in range (7, -1, -1):
                    # Output next bit
                    yield (ord (input) >> i) & 0x01

                input = file.read (1)

        while True:
            yield []

    def __readDHT (self, file):
        """ Read and compute the huffman tables. """

        # Read the marker length
        Lh = readWord (file)
        Lh -= 2
        while Lh > 0:
            huffsize = []
            huffval = []
            T = readByte (file)
            Th = T & 0x0F
            Tc = (T >> 4) & 0x0F
            #print "Lh: %d  Th: %d  Tc: %d" % (Lh, Th, Tc)
            Lh -= 1

            # Read how many symbols of each length
            # up to 16 bits
            for i in range (16):
                huffsize.append (readByte (file))
                Lh -= 1

            # Generate the huffman codes
            huffcode = generateHuffmanCodes (huffsize)
            #print "Huffcode", huffcode

            # Read the values that should be mapped to huffman codes
            for i in huffcode:
                #print i
                try:
                    huffval.append (readByte (file))
                    Lh -= 1
                except TypeError:
                    continue

            # Generate lookup tables
            if Tc == 0:
                self.huffman_dc_tables[Th] = mapHuffmanCodes (huffcode, huffval)
            else:
                self.huffman_ac_tables[Th] = mapHuffmanCodes (huffcode, huffval)

    def __readDNL (self, file):
        """ Read the DNL marker.  Changes the number of lines. """

        Ld = readWord (file)
        Ld -= 2
        NL = readWord (file)
        Ld -= 2

        X, Y, P = self.XYP

        if Y == 0:
            self.XYP = X, NL, P

    def __readDQT (self, file):
        """ Read the quantization table.  The table is in zigzag order. """

        Lq = readWord (file)
        Lq -= 2
        while Lq > 0:
            table = []
            Tq = readByte (file)
            Pq = Tq >> 4
            Tq &= 0xF
            Lq -= 1

            if Pq == 0:
                for i in range (64):
                    table.append (readByte (file))
                    Lq -= 1
            else:
                for i in range (64):
                    val = readWord (file)
                    table.append (val)
                    Lq -= 2

            self.q_table[Tq] = table

    def __readDU (self, comp_num):
        """ Read one data unit with component index comp_num. """

        data = []
        comp = self.component[comp_num]
        huff_tbl = self.huffman_dc_tables[comp['Td']]

        # Fill data with 64 coefficients
        while len (data) < 64:
            key = 0

            for bits in range (1, 17):
                key_len = []
                key <<= 1
                # Get one bit from bit_stream
                val = getBits (1, self.bit_stream)
                if val == []:
                    break
                key |= val
                # If huffman code exists
                if huff_tbl.has_key ((bits, key)):
                    key_len = huff_tbl[(bits, key)]
                    break

            # After getting the DC value switch to the AC table
            huff_tbl = self.huffman_ac_tables[comp['Ta']]

            if key_len == []:
                #print (bits, key, bin(key)), "key not found"
                break
            # If ZRL fill with 16 zero coefficients
            elif key_len == 0xF0:
                for i in range (16):
                    data.append (0)
                continue

            # If not DC coefficient
            if len (data) != 0:
                # If End of block
                if key_len == 0x00:
                    # Fill the rest of the DU with zeros
                    while len (data) < 64:
                        data.append (0)
                    break

                # The first part of the AC key_len is the number of leading
                # zeros
                for i in range (key_len >> 4):
                    if len (data) < 64:
                        data.append (0)
                key_len &= 0x0F

            if len (data) >= 64:
                break

            if key_len != 0:
                # The rest of key_len is the number of "additional" bits
                val = getBits (key_len, self.bit_stream)
                if val == []:
                    break
                # Decode the additional bits
                num = decodeBits (key_len, val)

                # Experimental, doesn't work right
                if len (data) == 0 and self.inline_dc:
                    # The DC coefficient value is added to the DC value from
                    # the corresponding DU in the previous MCU
                    num += self.dc[comp_num]
                    self.dc[comp_num] = num

                data.append (num)
            else:
                data.append (0)

        #if len(data) != 64:
            #print "Wrong size", len(data)

        return data

    def __readMCU (self):
        """ Read an MCU. """

        comp_num = mcu = range (self.num_components)

        # For each component
        for i in comp_num:
            comp = self.component[i + 1]
            mcu[i] = []
            # For each DU
            for j in range (comp['H'] * comp['V']):
                if not self.EOI:
                    mcu[i].append (self.__readDU (i + 1))

        self.mcus_read += 1

        return mcu

    def __readSOF (self, type, file):
        """ Read the start of frame marker. """

        Lf = readWord (file)            # Read the marker length
        Lf -= 2
        P = readByte (file)             # Read the sample precision
        Lf -= 1
        Y = readWord (file)             # Read number of lines
        Lf -= 2
        X = readWord (file)             # Read the number of samples per line
        Lf -= 2
        Nf = readByte (file)            # Read number of components
        Lf -= 1

        self.XYP = X, Y, P
        #print self.XYP

        while Lf > 0:
            C = readByte (file)         # Read component identifier
            V = readByte (file)         # Read sampling factors
            Tq = readByte (file)
            Lf -= 3
            H = V >> 4
            V &= 0xF
            # Assign horizontal & vertical sampling factors and qtable
            self.component[C] = { 'H' : H, 'V' : V, 'Tq' : Tq }

    def __readSOS (self, file):
        """ Read the start of scan marker. """

        Ls = readWord (file)
        Ls -= 2

        Ns = readByte (file)            # Read number of components in scan
        Ls -= 1

        for i in range (Ns):
            Cs = readByte (file)        # Read the scan component selector
            Ls -= 1
            Ta = readByte (file)        # Read the huffman table selectors
            Ls -= 1
            Td = Ta >> 4
            Ta &= 0xF
            # Assign the DC huffman table
            self.component[Cs]['Td'] = Td
            # Assign the AC huffman table
            self.component[Cs]['Ta'] = Ta

        Ss = readByte (file)            # Should be zero if baseline DCT
        Ls -= 1
        Se = readByte (file)            # Should be 63 if baseline DCT
        Ls -= 1
        A = readByte (file)             # Should be zero if baseline DCT
        Ls -= 1

        #print "Ns:%d Ss:%d Se:%d A:%02X" % (Ns, Ss, Se, A)
        self.num_components = Ns
        self.dc = [0 for i in range (self.num_components + 1)]

    def dequantize (self, mcu):
        """ Dequantize an MCU. """

        out = mcu

        # For each coefficient in each DU in each component, multiply by the
        # corresponding value in the quantization table.
        for c in range (len (out)):
            for du in range (len (out[c])):
                for i in range (len (out[c][du])):
                    out[c][du][i] *= self.q_table[self.component[c + 1]['Tq']][i]

        return out

def getHist(filename):
    try:
        import JPEG_MetaInfoPy
        hist, lowValue = JPEG_MetaInfoPy.generateHistogram(filename)
        return np.asarray(hist),np.asarray(range(lowValue,lowValue+len(hist)+1))
    except Exception as ex:
        logging.getLogger('maskgen').warn('External JPEG_MetaInfoPy failed: {}'.format(str(ex)))
        DC = JPEG_Reader().readDCT_Coeffs(filename)[0]
        minDC = min(DC)
        maxDC = max(DC)
        binCount = maxDC - minDC + 1
        return np.histogram (DC, bins=binCount,
                                                    range=(minDC, maxDC + 1))
class JPEG_View:
    def appliesTo (self, filename):
        return filename.lower ().endswith (('jpg', 'jpeg'))

    def draw (self, frame, filename):

        fig = plt.figure ();
        self._plotHistogram (fig, getHist(filename))
        canvas = FigureCanvasTkAgg (fig, frame)
        canvas.show ()
        canvas.get_tk_widget ().pack (side=BOTTOM, fill=BOTH, expand=True)

    def _labelSigma (self, figure, sigma):
        """ Add a label of the value of sigma to the histogram plot. """

        props = dict (boxstyle='round', facecolor='wheat', alpha=0.5)
        figure.text (0.25, 0.85, '$\sigma=%.2f$' % (sigma),
                     fontsize=14, verticalalignment='top', bbox=props)

class DCTView (JPEG_View):
    def screenName (self):
        return 'JPG DCT Histogram'

    def _plotHistogram (self, figure, histogram):
        ordinates, abscissae = histogram
        plt.bar (abscissae[:-1], ordinates, 1);
        self._labelSigma (figure, ordinates.std ())

class FFT_DCTView (JPEG_View):
    def screenName (self):
        return 'FFT(JPG DCT Histogram)'

    def _plotHistogram (self, figure, histogram):

        # Calculate the DFT of the zero-meaned histogram values.  The n/2+1
        # positive frequencies are returned by rfft.  Mirror the result back
        # into ordinates.
        #
        mean = histogram[0].mean ()
        posFreqs = abs (np.fft.rfft ([i - mean for i in histogram[0]]))
        ordinates = list (reversed (posFreqs))
        ordinates.extend (posFreqs[1:])
        n = len (posFreqs)
        abscissae = range (1 - n, n)

        plt.plot (abscissae, ordinates, 'k')
        plt.plot (abscissae, self.__hat (ordinates), 'r')
        self._labelSigma (figure, np.std (ordinates))

    def __hat (self, data):
        length = len (data)
        intercept1 = int (length * 0.425)
        intercept2 = int (length * 0.575)
        amp = max (data)
        threshold = amp * 0.15

        arr = np.full (length, threshold)
        arr[intercept1:intercept2] = amp

        return arr

if __name__ == "__main__":
    DCTView ().draw (None, sys.argv[1])
    FFT_DCTView ().draw (None, sys.argv[1])


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import random
from specparser import get_settings_from_file
from pprint import pprint
import csv
from sklearn.svm import LinearSVC
import numpy as np
from sklearn.externals import joblib

import matplotlib
matplotlib.use('Agg')

import datetime
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt

def drawPrecisionRecall(X,Y,output_file):
    pdf = PdfPages(output_file)
    plt.figure(figsize=(len(Y), len(X)))
    plt.plot(Y, X, 'r-o')
    plt.title('Precision/Recall')
    pdf.savefig()  # saves the current figure into a pdf page
    plt.close()
    pdf.close()

def readcolumn(filename):
    column = []
    with open(filename,"r") as fconcl:
        for line in fconcl:
            column.append(line.rstrip('\n'))
    return  column

def main():
    settings = get_settings_from_file("spec.json")
    print(settings)
    X = np.genfromtxt(settings.Input.X, delimiter=',', skip_header=1)
    svc = joblib.load(settings.Input.MODEL)
    Y_out = svc.predict(X)
    Y_list = [Y_out]
    np.savetxt("./conclusion.csv", Y_out, fmt="%d", delimiter=",")

    conclusion = readcolumn("./conclusion.csv")
    label = readcolumn(settings.Input.Y)

    precision_list = []
    recall_list = []

    hits = 0
    for i in range(len(label)):
        if conclusion[i] == label[i]:
            hits+=1
            precision_list.append(1.0*hits/(i+1))
            recall_list.append(1.0*hits/(len(label)))

    drawPrecisionRecall(precision_list,recall_list,settings.Output.report)
    print("Done")

if __name__ == "__main__":
    main()


"""
Oceanographic Profiles and T-S Diagrams
=======================================

This example demonstrates how to plot vertical profiles of different
variables in the same axes, and how to make a scatter plot of two
variables. There is an oceanographic theme but the same techniques are
equally applicable to atmospheric or other kinds of data.

The data used are profiles of potential temperature and salinity in the
Equatorial and South Atlantic, output from an ocean model.

The y-axis of the first plot produced will be automatically inverted due to the
presence of the attribute positive=down on the depth coordinate. This means
depth values intuitively increase downward on the y-axis.

"""

import matplotlib.pyplot as plt

import iris
import iris.iterate
import iris.plot as iplt

def main():
    # Load the gridded temperature and salinity data.
    fname = iris.sample_data_path("atlantic_profiles.nc")
    cubes = iris.load(fname)
    (theta,) = cubes.extract("sea_water_potential_temperature")
    (salinity,) = cubes.extract("sea_water_practical_salinity")

    # Extract profiles of temperature and salinity from a particular point in
    # the southern portion of the domain, and limit the depth of the profile
    # to 1000m.
    lon_cons = iris.Constraint(longitude=330.5)
    lat_cons = iris.Constraint(latitude=lambda l: -10 < l < -9)
    depth_cons = iris.Constraint(depth=lambda d: d <= 1000)
    theta_1000m = theta.extract(depth_cons & lon_cons & lat_cons)
    salinity_1000m = salinity.extract(depth_cons & lon_cons & lat_cons)

    # Plot these profiles on the same set of axes. Depth is automatically
    # recognised as a vertical coordinate and placed on the y-axis.
    # The first plot is in the default axes. We'll use the same color for the
    # curve and its axes/tick labels.
    plt.figure(figsize=(5, 6))
    temperature_color = (0.3, 0.4, 0.5)
    ax1 = plt.gca()
    iplt.plot(
        theta_1000m,
        linewidth=2,
        color=temperature_color,
        alpha=0.75,
    )
    ax1.set_xlabel("Potential Temperature / K", color=temperature_color)
    ax1.set_ylabel("Depth / m")
    for ticklabel in ax1.get_xticklabels():
        ticklabel.set_color(temperature_color)

    # To plot salinity in the same axes we use twiny(). We'll use a different
    # color to identify salinity.
    salinity_color = (0.6, 0.1, 0.15)
    ax2 = plt.gca().twiny()
    iplt.plot(
        salinity_1000m,
        linewidth=2,
        color=salinity_color,
        alpha=0.75,
    )
    ax2.set_xlabel("Salinity / PSU", color=salinity_color)
    for ticklabel in ax2.get_xticklabels():
        ticklabel.set_color(salinity_color)
    plt.tight_layout()
    iplt.show()

    # Now plot a T-S diagram using scatter. We'll use all the profiles here,
    # and each point will be coloured according to its depth.
    plt.figure(figsize=(6, 6))
    depth_values = theta.coord("depth").points
    for s, t in iris.iterate.izip(salinity, theta, coords="depth"):
        iplt.scatter(s, t, c=depth_values, marker="+", cmap="RdYlBu_r")
    ax = plt.gca()
    ax.set_xlabel("Salinity / PSU")
    ax.set_ylabel("Potential Temperature / K")
    cb = plt.colorbar(orientation="horizontal")
    cb.set_label("Depth / m")
    plt.tight_layout()
    iplt.show()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Script for generating mock EPrime test data
"""

import pandas as pd
import numpy as np
import io
pd.set_option('display.max_rows', 50)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
from pypsych.config import Config

def generate_mock_eprime_data(config_path, task_name, begaze_data, sched_path):
    """Generate mock eprime data based on mock begaze data."""
    superconfig = Config(path=config_path)
    superconfig.load()
    config = superconfig.get_subconfig(task_name, 'EPrime')

    bg = begaze_data['merged_labels'][['Condition', 'ID']]
    ed = np.random.randint(0, 10, (bg.shape[0], len(config['channels'])))
    ep = pd.DataFrame(data=ed, index=bg.index, columns=config['channels'])
    df = pd.concat([bg, ep], axis=1, join='inner')

    df.rename(columns={'ID': 'Img'}, inplace=True)

    result = []
    for _, row in df.iterrows():
        props = ["\t" + str(idx) + ': ' + str(val)
                 for idx, val in zip(list(row.index), list(row))]
        result.append("\n\n".join(props))
    result = ('\n\n\t*** LogFrame End ***\n\n'
              '\tLevel: 2\n\n'
              '\t*** LogFrame Start ***\n\n').join(result)
    prestring = ('*** Header Start ***\n\n'
                 'GARBAGE\n\n'
                 '*** Header End ***\n\n'
                 '\tLevel: 2\n\n'
                 '\t*** LogFrame Start ***\n\n')
    result = prestring + result + '\n\n\t*** LogFrame End ***'
    return {'df': df, 'raw': result}

def save_mock_eprime_data(output_path, data, subject_id, task_order, task_name):
    """Save the mock eprime files to output_path."""
    base_path = ''.join([output_path,
                         task_name,
                         '_',
                         str(subject_id),
                         str(task_order)])
    raw_path = ''.join([base_path, '_eprime.txt'])
    df_path = ''.join([base_path, '_eprime_df.txt'])

    with io.open(raw_path, 'w', encoding="utf-16") as f:
        f.write(unicode(data['raw']))

    data['df'].to_csv(df_path, sep="\t")

    pass


# -*- coding: utf-8 -*-

# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Robert Layton <robertlayton@gmail.com>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
#          Philippe Gervais <philippe.gervais@inria.fr>
#          Lars Buitinck <larsmans@gmail.com>
#          Joel Nothman <joel.nothman@gmail.com>
# License: BSD 3 clause

import itertools

import numpy as np
from scipy.spatial import distance
from scipy.sparse import csr_matrix
from scipy.sparse import issparse

from ..utils import check_array
from ..utils import gen_even_slices
from ..utils import gen_batches
from ..utils.fixes import partial
from ..utils.extmath import row_norms, safe_sparse_dot
from ..preprocessing import normalize
from ..externals.joblib import Parallel
from ..externals.joblib import delayed
from ..externals.joblib.parallel import cpu_count

from .pairwise_fast import _chi2_kernel_fast, _sparse_manhattan

# Utility Functions
def _return_float_dtype(X, Y):
    """
    1. If dtype of X and Y is float32, then dtype float32 is returned.
    2. Else dtype float is returned.
    """
    if not issparse(X) and not isinstance(X, np.ndarray):
        X = np.asarray(X)

    if Y is None:
        Y_dtype = X.dtype
    elif not issparse(Y) and not isinstance(Y, np.ndarray):
        Y = np.asarray(Y)
        Y_dtype = Y.dtype
    else:
        Y_dtype = Y.dtype

    if X.dtype == Y_dtype == np.float32:
        dtype = np.float32
    else:
        dtype = np.float

    return X, Y, dtype

def check_pairwise_arrays(X, Y, precomputed=False):
    """ Set X and Y appropriately and checks inputs

    If Y is None, it is set as a pointer to X (i.e. not a copy).
    If Y is given, this does not happen.
    All distance metrics should use this function first to assert that the
    given parameters are correct and safe to use.

    Specifically, this function first ensures that both X and Y are arrays,
    then checks that they are at least two dimensional while ensuring that
    their elements are floats. Finally, the function checks that the size
    of the second dimension of the two arrays is equal, or the equivalent
    check for a precomputed distance matrix.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples_a, n_features)

    Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)

    precomputed : bool
        True if X is to be treated as precomputed distances to the samples in
        Y.

    Returns
    -------
    safe_X : {array-like, sparse matrix}, shape (n_samples_a, n_features)
        An array equal to X, guaranteed to be a numpy array.

    safe_Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)
        An array equal to Y if Y was not None, guaranteed to be a numpy array.
        If Y was None, safe_Y will be a pointer to X.

    """
    X, Y, dtype = _return_float_dtype(X, Y)

    if Y is X or Y is None:
        X = Y = check_array(X, accept_sparse='csr', dtype=dtype)
    else:
        X = check_array(X, accept_sparse='csr', dtype=dtype)
        Y = check_array(Y, accept_sparse='csr', dtype=dtype)

    if precomputed:
        if X.shape[1] != Y.shape[0]:
            raise ValueError("Precomputed metric requires shape "
                             "(n_queries, n_indexed). Got (%d, %d) "
                             "for %d indexed." %
                             (X.shape[0], X.shape[1], Y.shape[0]))
    elif X.shape[1] != Y.shape[1]:
        raise ValueError("Incompatible dimension for X and Y matrices: "
                         "X.shape[1] == %d while Y.shape[1] == %d" % (
                             X.shape[1], Y.shape[1]))

    return X, Y

def check_paired_arrays(X, Y):
    """ Set X and Y appropriately and checks inputs for paired distances

    All paired distance metrics should use this function first to assert that
    the given parameters are correct and safe to use.

    Specifically, this function first ensures that both X and Y are arrays,
    then checks that they are at least two dimensional while ensuring that
    their elements are floats. Finally, the function checks that the size
    of the dimensions of the two arrays are equal.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples_a, n_features)

    Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)

    Returns
    -------
    safe_X : {array-like, sparse matrix}, shape (n_samples_a, n_features)
        An array equal to X, guaranteed to be a numpy array.

    safe_Y : {array-like, sparse matrix}, shape (n_samples_b, n_features)
        An array equal to Y if Y was not None, guaranteed to be a numpy array.
        If Y was None, safe_Y will be a pointer to X.

    """
    X, Y = check_pairwise_arrays(X, Y)
    if X.shape != Y.shape:
        raise ValueError("X and Y should be of same shape. They were "
                         "respectively %r and %r long." % (X.shape, Y.shape))
    return X, Y

# Pairwise distances
def euclidean_distances(X, Y=None, Y_norm_squared=None, squared=False,
                        X_norm_squared=None):
    """
    Considering the rows of X (and Y=X) as vectors, compute the
    distance matrix between each pair of vectors.

    For efficiency reasons, the euclidean distance between a pair of row
    vector x and y is computed as::

        dist(x, y) = sqrt(dot(x, x) - 2 * dot(x, y) + dot(y, y))

    This formulation has two advantages over other ways of computing distances.
    First, it is computationally efficient when dealing with sparse data.
    Second, if one argument varies but the other remains unchanged, then
    `dot(x, x)` and/or `dot(y, y)` can be pre-computed.

    However, this is not the most precise way of doing this computation, and
    the distance matrix returned by this function may not be exactly
    symmetric as required by, e.g., ``scipy.spatial.distance`` functions.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape (n_samples_1, n_features)

    Y : {array-like, sparse matrix}, shape (n_samples_2, n_features)

    Y_norm_squared : array-like, shape (n_samples_2, ), optional
        Pre-computed dot-products of vectors in Y (e.g.,
        ``(Y**2).sum(axis=1)``)

    squared : boolean, optional
        Return squared Euclidean distances.

    X_norm_squared : array-like, shape = [n_samples_1], optional
        Pre-computed dot-products of vectors in X (e.g.,
        ``(X**2).sum(axis=1)``)

    Returns
    -------
    distances : {array, sparse matrix}, shape (n_samples_1, n_samples_2)

    Examples
    --------
    >>> from sklearn.metrics.pairwise import euclidean_distances
    >>> X = [[0, 1], [1, 1]]
    >>> # distance between rows of X
    >>> euclidean_distances(X, X)
    array([[ 0.,  1.],
           [ 1.,  0.]])
    >>> # get distance to origin
    >>> euclidean_distances(X, [[0, 0]])
    array([[ 1.        ],
           [ 1.41421356]])

    See also
    --------
    paired_distances : distances betweens pairs of elements of X and Y.
    """
    X, Y = check_pairwise_arrays(X, Y)

    if X_norm_squared is not None:
        XX = check_array(X_norm_squared)
        if XX.shape == (1, X.shape[0]):
            XX = XX.T
        elif XX.shape != (X.shape[0], 1):
            raise ValueError(
                "Incompatible dimensions for X and X_norm_squared")
    else:
        XX = row_norms(X, squared=True)[:, np.newaxis]

    if X is Y:  # shortcut in the common case euclidean_distances(X, X)
        YY = XX.T
    elif Y_norm_squared is not None:
        YY = np.atleast_2d(Y_norm_squared)

        if YY.shape != (1, Y.shape[0]):
            raise ValueError(
                "Incompatible dimensions for Y and Y_norm_squared")
    else:
        YY = row_norms(Y, squared=True)[np.newaxis, :]

    distances = safe_sparse_dot(X, Y.T, dense_output=True)
    distances *= -2
    distances += XX
    distances += YY
    np.maximum(distances, 0, out=distances)

    if X is Y:
        # Ensure that distances between vectors and themselves are set to 0.0.
        # This may not be the case due to floating point rounding errors.
        distances.flat[::distances.shape[0] + 1] = 0.0

    return distances if squared else np.sqrt(distances, out=distances)

def pairwise_distances_argmin_min(X, Y, axis=1, metric="euclidean",
                                  batch_size=500, metric_kwargs=None):
    """Compute minimum distances between one point and a set of points.

    This function computes for each row in X, the index of the row of Y which
    is closest (according to the specified distance). The minimal distances are
    also returned.

    This is mostly equivalent to calling:

        (pairwise_distances(X, Y=Y, metric=metric).argmin(axis=axis),
         pairwise_distances(X, Y=Y, metric=metric).min(axis=axis))

    but uses much less memory, and is faster for large arrays.

    Parameters
    ----------
    X, Y : {array-like, sparse matrix}
        Arrays containing points. Respective shapes (n_samples1, n_features)
        and (n_samples2, n_features)

    batch_size : integer
        To reduce memory consumption over the naive solution, data are
        processed in batches, comprising batch_size rows of X and
        batch_size rows of Y. The default value is quite conservative, but
        can be changed for fine-tuning. The larger the number, the larger the
        memory usage.

    metric : string or callable, default 'euclidean'
        metric to use for distance computation. Any metric from scikit-learn
        or scipy.spatial.distance can be used.

        If metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays as input and return one value indicating the
        distance between them. This works for Scipy's metrics, but is less
        efficient than passing the metric name as a string.

        Distance matrices are not supported.

        Valid values for metric are:

        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
          'manhattan']

        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
          'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
          'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath',
          'sqeuclidean', 'yule']

        See the documentation for scipy.spatial.distance for details on these
        metrics.

    metric_kwargs : dict, optional
        Keyword arguments to pass to specified metric function.

    axis : int, optional, default 1
        Axis along which the argmin and distances are to be computed.

    Returns
    -------
    argmin : numpy.ndarray
        Y[argmin[i], :] is the row in Y that is closest to X[i, :].

    distances : numpy.ndarray
        distances[i] is the distance between the i-th row in X and the
        argmin[i]-th row in Y.

    See also
    --------
    sklearn.metrics.pairwise_distances
    sklearn.metrics.pairwise_distances_argmin
    """
    dist_func = None
    if metric in PAIRWISE_DISTANCE_FUNCTIONS:
        dist_func = PAIRWISE_DISTANCE_FUNCTIONS[metric]
    elif not callable(metric) and not isinstance(metric, str):
        raise ValueError("'metric' must be a string or a callable")

    X, Y = check_pairwise_arrays(X, Y)

    if metric_kwargs is None:
        metric_kwargs = {}

    if axis == 0:
        X, Y = Y, X

    # Allocate output arrays
    indices = np.empty(X.shape[0], dtype=np.intp)
    values = np.empty(X.shape[0])
    values.fill(np.infty)

    for chunk_x in gen_batches(X.shape[0], batch_size):
        X_chunk = X[chunk_x, :]

        for chunk_y in gen_batches(Y.shape[0], batch_size):
            Y_chunk = Y[chunk_y, :]

            if dist_func is not None:
                if metric == 'euclidean':  # special case, for speed
                    d_chunk = safe_sparse_dot(X_chunk, Y_chunk.T,
                                              dense_output=True)
                    d_chunk *= -2
                    d_chunk += row_norms(X_chunk, squared=True)[:, np.newaxis]
                    d_chunk += row_norms(Y_chunk, squared=True)[np.newaxis, :]
                    np.maximum(d_chunk, 0, d_chunk)
                else:
                    d_chunk = dist_func(X_chunk, Y_chunk, **metric_kwargs)
            else:
                d_chunk = pairwise_distances(X_chunk, Y_chunk,
                                             metric=metric, **metric_kwargs)

            # Update indices and minimum values using chunk
            min_indices = d_chunk.argmin(axis=1)
            min_values = d_chunk[np.arange(chunk_x.stop - chunk_x.start),
                                 min_indices]

            flags = values[chunk_x] > min_values
            indices[chunk_x][flags] = min_indices[flags] + chunk_y.start
            values[chunk_x][flags] = min_values[flags]

    if metric == "euclidean" and not metric_kwargs.get("squared", False):
        np.sqrt(values, values)
    return indices, values

def pairwise_distances_argmin(X, Y, axis=1, metric="euclidean",
                              batch_size=500, metric_kwargs=None):
    """Compute minimum distances between one point and a set of points.

    This function computes for each row in X, the index of the row of Y which
    is closest (according to the specified distance).

    This is mostly equivalent to calling:

        pairwise_distances(X, Y=Y, metric=metric).argmin(axis=axis)

    but uses much less memory, and is faster for large arrays.

    This function works with dense 2D arrays only.

    Parameters
    ----------
    X : array-like
        Arrays containing points. Respective shapes (n_samples1, n_features)
        and (n_samples2, n_features)

    Y : array-like
        Arrays containing points. Respective shapes (n_samples1, n_features)
        and (n_samples2, n_features)

    batch_size : integer
        To reduce memory consumption over the naive solution, data are
        processed in batches, comprising batch_size rows of X and
        batch_size rows of Y. The default value is quite conservative, but
        can be changed for fine-tuning. The larger the number, the larger the
        memory usage.

    metric : string or callable
        metric to use for distance computation. Any metric from scikit-learn
        or scipy.spatial.distance can be used.

        If metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays as input and return one value indicating the
        distance between them. This works for Scipy's metrics, but is less
        efficient than passing the metric name as a string.

        Distance matrices are not supported.

        Valid values for metric are:

        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
          'manhattan']

        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
          'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
          'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath',
          'sqeuclidean', 'yule']

        See the documentation for scipy.spatial.distance for details on these
        metrics.

    metric_kwargs : dict
        keyword arguments to pass to specified metric function.

    axis : int, optional, default 1
        Axis along which the argmin and distances are to be computed.

    Returns
    -------
    argmin : numpy.ndarray
        Y[argmin[i], :] is the row in Y that is closest to X[i, :].

    See also
    --------
    sklearn.metrics.pairwise_distances
    sklearn.metrics.pairwise_distances_argmin_min
    """
    if metric_kwargs is None:
        metric_kwargs = {}

    return pairwise_distances_argmin_min(X, Y, axis, metric, batch_size,
                                         metric_kwargs)[0]

def manhattan_distances(X, Y=None, sum_over_features=True,
                        size_threshold=5e8):
    """ Compute the L1 distances between the vectors in X and Y.

    With sum_over_features equal to False it returns the componentwise
    distances.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array_like
        An array with shape (n_samples_X, n_features).

    Y : array_like, optional
        An array with shape (n_samples_Y, n_features).

    sum_over_features : bool, default=True
        If True the function returns the pairwise distance matrix
        else it returns the componentwise L1 pairwise-distances.
        Not supported for sparse matrix inputs.

    size_threshold : int, default=5e8
        Unused parameter.

    Returns
    -------
    D : array
        If sum_over_features is False shape is
        (n_samples_X * n_samples_Y, n_features) and D contains the
        componentwise L1 pairwise-distances (ie. absolute difference),
        else shape is (n_samples_X, n_samples_Y) and D contains
        the pairwise L1 distances.

    Examples
    --------
    >>> from sklearn.metrics.pairwise import manhattan_distances
    >>> manhattan_distances([[3]], [[3]])#doctest:+ELLIPSIS
    array([[ 0.]])
    >>> manhattan_distances([[3]], [[2]])#doctest:+ELLIPSIS
    array([[ 1.]])
    >>> manhattan_distances([[2]], [[3]])#doctest:+ELLIPSIS
    array([[ 1.]])
    >>> manhattan_distances([[1, 2], [3, 4]],\
         [[1, 2], [0, 3]])#doctest:+ELLIPSIS
    array([[ 0.,  2.],
           [ 4.,  4.]])
    >>> import numpy as np
    >>> X = np.ones((1, 2))
    >>> y = 2 * np.ones((2, 2))
    >>> manhattan_distances(X, y, sum_over_features=False)#doctest:+ELLIPSIS
    array([[ 1.,  1.],
           [ 1.,  1.]]...)
    """
    X, Y = check_pairwise_arrays(X, Y)

    if issparse(X) or issparse(Y):
        if not sum_over_features:
            raise TypeError("sum_over_features=%r not supported"
                            " for sparse matrices" % sum_over_features)

        X = csr_matrix(X, copy=False)
        Y = csr_matrix(Y, copy=False)
        D = np.zeros((X.shape[0], Y.shape[0]))
        _sparse_manhattan(X.data, X.indices, X.indptr,
                          Y.data, Y.indices, Y.indptr,
                          X.shape[1], D)
        return D

    if sum_over_features:
        return distance.cdist(X, Y, 'cityblock')

    D = X[:, np.newaxis, :] - Y[np.newaxis, :, :]
    D = np.abs(D, D)
    return D.reshape((-1, X.shape[1]))

def cosine_distances(X, Y=None):
    """Compute cosine distance between samples in X and Y.

    Cosine distance is defined as 1.0 minus the cosine similarity.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array_like, sparse matrix
        with shape (n_samples_X, n_features).

    Y : array_like, sparse matrix (optional)
        with shape (n_samples_Y, n_features).

    Returns
    -------
    distance matrix : array
        An array with shape (n_samples_X, n_samples_Y).

    See also
    --------
    sklearn.metrics.pairwise.cosine_similarity
    scipy.spatial.distance.cosine (dense matrices only)
    """
    # 1.0 - cosine_similarity(X, Y) without copy
    S = cosine_similarity(X, Y)
    S *= -1
    S += 1
    return S

# Paired distances
def paired_euclidean_distances(X, Y):
    """
    Computes the paired euclidean distances between X and Y

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    Y : array-like, shape (n_samples, n_features)

    Returns
    -------
    distances : ndarray (n_samples, )
    """
    X, Y = check_paired_arrays(X, Y)
    return row_norms(X - Y)

def paired_manhattan_distances(X, Y):
    """Compute the L1 distances between the vectors in X and Y.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    Y : array-like, shape (n_samples, n_features)

    Returns
    -------
    distances : ndarray (n_samples, )
    """
    X, Y = check_paired_arrays(X, Y)
    diff = X - Y
    if issparse(diff):
        diff.data = np.abs(diff.data)
        return np.squeeze(np.array(diff.sum(axis=1)))
    else:
        return np.abs(diff).sum(axis=-1)

def paired_cosine_distances(X, Y):
    """
    Computes the paired cosine distances between X and Y

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array-like, shape (n_samples, n_features)

    Y : array-like, shape (n_samples, n_features)

    Returns
    -------
    distances : ndarray, shape (n_samples, )

    Notes
    ------
    The cosine distance is equivalent to the half the squared
    euclidean distance if each sample is normalized to unit norm
    """
    X, Y = check_paired_arrays(X, Y)
    return .5 * row_norms(normalize(X) - normalize(Y), squared=True)

PAIRED_DISTANCES = {
    'cosine': paired_cosine_distances,
    'euclidean': paired_euclidean_distances,
    'l2': paired_euclidean_distances,
    'l1': paired_manhattan_distances,
    'manhattan': paired_manhattan_distances,
    'cityblock': paired_manhattan_distances}

def paired_distances(X, Y, metric="euclidean", **kwds):
    """
    Computes the paired distances between X and Y.

    Computes the distances between (X[0], Y[0]), (X[1], Y[1]), etc...

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : ndarray (n_samples, n_features)
        Array 1 for distance computation.

    Y : ndarray (n_samples, n_features)
        Array 2 for distance computation.

    metric : string or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        specified in PAIRED_DISTANCES, including "euclidean",
        "manhattan", or "cosine".
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them.

    Returns
    -------
    distances : ndarray (n_samples, )

    Examples
    --------
    >>> from sklearn.metrics.pairwise import paired_distances
    >>> X = [[0, 1], [1, 1]]
    >>> Y = [[0, 1], [2, 1]]
    >>> paired_distances(X, Y)
    array([ 0.,  1.])

    See also
    --------
    pairwise_distances : pairwise distances.
    """

    if metric in PAIRED_DISTANCES:
        func = PAIRED_DISTANCES[metric]
        return func(X, Y)
    elif callable(metric):
        # Check the matrix first (it is usually done by the metric)
        X, Y = check_paired_arrays(X, Y)
        distances = np.zeros(len(X))
        for i in range(len(X)):
            distances[i] = metric(X[i], Y[i])
        return distances
    else:
        raise ValueError('Unknown distance %s' % metric)

# Kernels
def linear_kernel(X, Y=None):
    """
    Compute the linear kernel between X and Y.

    Read more in the :ref:`User Guide <linear_kernel>`.

    Parameters
    ----------
    X : array of shape (n_samples_1, n_features)

    Y : array of shape (n_samples_2, n_features)

    Returns
    -------
    Gram matrix : array of shape (n_samples_1, n_samples_2)
    """
    X, Y = check_pairwise_arrays(X, Y)
    return safe_sparse_dot(X, Y.T, dense_output=True)

def polynomial_kernel(X, Y=None, degree=3, gamma=None, coef0=1):
    """
    Compute the polynomial kernel between X and Y::

        K(X, Y) = (gamma <X, Y> + coef0)^degree

    Read more in the :ref:`User Guide <polynomial_kernel>`.

    Parameters
    ----------
    X : ndarray of shape (n_samples_1, n_features)

    Y : ndarray of shape (n_samples_2, n_features)

    coef0 : int, default 1

    degree : int, default 3

    Returns
    -------
    Gram matrix : array of shape (n_samples_1, n_samples_2)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = safe_sparse_dot(X, Y.T, dense_output=True)
    K *= gamma
    K += coef0
    K **= degree
    return K

def sigmoid_kernel(X, Y=None, gamma=None, coef0=1):
    """
    Compute the sigmoid kernel between X and Y::

        K(X, Y) = tanh(gamma <X, Y> + coef0)

    Read more in the :ref:`User Guide <sigmoid_kernel>`.

    Parameters
    ----------
    X : ndarray of shape (n_samples_1, n_features)

    Y : ndarray of shape (n_samples_2, n_features)

    coef0 : int, default 1

    Returns
    -------
    Gram matrix: array of shape (n_samples_1, n_samples_2)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = safe_sparse_dot(X, Y.T, dense_output=True)
    K *= gamma
    K += coef0
    np.tanh(K, K)   # compute tanh in-place
    return K

def rbf_kernel(X, Y=None, gamma=None):
    """
    Compute the rbf (gaussian) kernel between X and Y::

        K(x, y) = exp(-gamma ||x-y||^2)

    for each pair of rows x in X and y in Y.

    Read more in the :ref:`User Guide <rbf_kernel>`.

    Parameters
    ----------
    X : array of shape (n_samples_X, n_features)

    Y : array of shape (n_samples_Y, n_features)

    gamma : float

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = euclidean_distances(X, Y, squared=True)
    K *= -gamma
    np.exp(K, K)    # exponentiate K in-place
    return K

def laplacian_kernel(X, Y=None, gamma=None):
    """Compute the laplacian kernel between X and Y.

    The laplacian kernel is defined as::

        K(x, y) = exp(-gamma ||x-y||_1)

    for each pair of rows x in X and y in Y.
    Read more in the :ref:`User Guide <laplacian_kernel>`.

    .. versionadded:: 0.17

    Parameters
    ----------
    X : array of shape (n_samples_X, n_features)
    Y : array of shape (n_samples_Y, n_features)
    gamma : float

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)
    """
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = -gamma * manhattan_distances(X, Y)
    np.exp(K, K)    # exponentiate K in-place
    return K

def cosine_similarity(X, Y=None, dense_output=True):
    """Compute cosine similarity between samples in X and Y.

    Cosine similarity, or the cosine kernel, computes similarity as the
    normalized dot product of X and Y:

        K(X, Y) = <X, Y> / (||X||*||Y||)

    On L2-normalized data, this function is equivalent to linear_kernel.

    Read more in the :ref:`User Guide <cosine_similarity>`.

    Parameters
    ----------
    X : ndarray or sparse array, shape: (n_samples_X, n_features)
        Input data.

    Y : ndarray or sparse array, shape: (n_samples_Y, n_features)
        Input data. If ``None``, the output will be the pairwise
        similarities between all samples in ``X``.

    dense_output : boolean (optional), default True
        Whether to return dense output even when the input is sparse. If
        ``False``, the output is sparse if both input arrays are sparse.

        .. versionadded:: 0.17
           parameter *dense_output* for sparse output.

    Returns
    -------
    kernel matrix : array
        An array with shape (n_samples_X, n_samples_Y).
    """
    # to avoid recursive import

    X, Y = check_pairwise_arrays(X, Y)

    X_normalized = normalize(X, copy=True)
    if X is Y:
        Y_normalized = X_normalized
    else:
        Y_normalized = normalize(Y, copy=True)

    K = safe_sparse_dot(X_normalized, Y_normalized.T, dense_output=dense_output)

    return K

def additive_chi2_kernel(X, Y=None):
    """Computes the additive chi-squared kernel between observations in X and Y

    The chi-squared kernel is computed between each pair of rows in X and Y.  X
    and Y have to be non-negative. This kernel is most commonly applied to
    histograms.

    The chi-squared kernel is given by::

        k(x, y) = -Sum [(x - y)^2 / (x + y)]

    It can be interpreted as a weighted difference per entry.

    Read more in the :ref:`User Guide <chi2_kernel>`.

    Notes
    -----
    As the negative of a distance, this kernel is only conditionally positive
    definite.

    Parameters
    ----------
    X : array-like of shape (n_samples_X, n_features)

    Y : array of shape (n_samples_Y, n_features)

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)

    References
    ----------
    * Zhang, J. and Marszalek, M. and Lazebnik, S. and Schmid, C.
      Local features and kernels for classification of texture and object
      categories: A comprehensive study
      International Journal of Computer Vision 2007
      http://research.microsoft.com/en-us/um/people/manik/projects/trade-off/papers/ZhangIJCV06.pdf

    See also
    --------
    chi2_kernel : The exponentiated version of the kernel, which is usually
        preferable.

    sklearn.kernel_approximation.AdditiveChi2Sampler : A Fourier approximation
        to this kernel.
    """
    if issparse(X) or issparse(Y):
        raise ValueError("additive_chi2 does not support sparse matrices.")
    X, Y = check_pairwise_arrays(X, Y)
    if (X < 0).any():
        raise ValueError("X contains negative values.")
    if Y is not X and (Y < 0).any():
        raise ValueError("Y contains negative values.")

    result = np.zeros((X.shape[0], Y.shape[0]), dtype=X.dtype)
    _chi2_kernel_fast(X, Y, result)
    return result

def chi2_kernel(X, Y=None, gamma=1.):
    """Computes the exponential chi-squared kernel X and Y.

    The chi-squared kernel is computed between each pair of rows in X and Y.  X
    and Y have to be non-negative. This kernel is most commonly applied to
    histograms.

    The chi-squared kernel is given by::

        k(x, y) = exp(-gamma Sum [(x - y)^2 / (x + y)])

    It can be interpreted as a weighted difference per entry.

    Read more in the :ref:`User Guide <chi2_kernel>`.

    Parameters
    ----------
    X : array-like of shape (n_samples_X, n_features)

    Y : array of shape (n_samples_Y, n_features)

    gamma : float, default=1.
        Scaling parameter of the chi2 kernel.

    Returns
    -------
    kernel_matrix : array of shape (n_samples_X, n_samples_Y)

    References
    ----------
    * Zhang, J. and Marszalek, M. and Lazebnik, S. and Schmid, C.
      Local features and kernels for classification of texture and object
      categories: A comprehensive study
      International Journal of Computer Vision 2007
      http://research.microsoft.com/en-us/um/people/manik/projects/trade-off/papers/ZhangIJCV06.pdf

    See also
    --------
    additive_chi2_kernel : The additive version of this kernel

    sklearn.kernel_approximation.AdditiveChi2Sampler : A Fourier approximation
        to the additive version of this kernel.
    """
    K = additive_chi2_kernel(X, Y)
    K *= gamma
    return np.exp(K, K)

# Helper functions - distance
PAIRWISE_DISTANCE_FUNCTIONS = {
    # If updating this dictionary, update the doc in both distance_metrics()
    # and also in pairwise_distances()!
    'cityblock': manhattan_distances,
    'cosine': cosine_distances,
    'euclidean': euclidean_distances,
    'l2': euclidean_distances,
    'l1': manhattan_distances,
    'manhattan': manhattan_distances,
    'precomputed': None,  # HACK: precomputed is always allowed, never called
}

def distance_metrics():
    """Valid metrics for pairwise_distances.

    This function simply returns the valid pairwise distance metrics.
    It exists to allow for a description of the mapping for
    each of the valid strings.

    The valid distance metrics, and the function they map to, are:

    ============     ====================================
    metric           Function
    ============     ====================================
    'cityblock'      metrics.pairwise.manhattan_distances
    'cosine'         metrics.pairwise.cosine_distances
    'euclidean'      metrics.pairwise.euclidean_distances
    'l1'             metrics.pairwise.manhattan_distances
    'l2'             metrics.pairwise.euclidean_distances
    'manhattan'      metrics.pairwise.manhattan_distances
    ============     ====================================

    Read more in the :ref:`User Guide <metrics>`.

    """
    return PAIRWISE_DISTANCE_FUNCTIONS

def _parallel_pairwise(X, Y, func, n_jobs, **kwds):
    """Break the pairwise matrix in n_jobs even slices
    and compute them in parallel"""
    if n_jobs < 0:
        n_jobs = max(cpu_count() + 1 + n_jobs, 1)

    if Y is None:
        Y = X

    if n_jobs == 1:
        # Special case to avoid picklability checks in delayed
        return func(X, Y, **kwds)

    # TODO: in some cases, backend='threading' may be appropriate
    fd = delayed(func)
    ret = Parallel(n_jobs=n_jobs, verbose=0)(
        fd(X, Y[s], **kwds)
        for s in gen_even_slices(Y.shape[0], n_jobs))

    return np.hstack(ret)

def _pairwise_callable(X, Y, metric, **kwds):
    """Handle the callable case for pairwise_{distances,kernels}
    """
    X, Y = check_pairwise_arrays(X, Y)

    if X is Y:
        # Only calculate metric for upper triangle
        out = np.zeros((X.shape[0], Y.shape[0]), dtype='float')
        iterator = itertools.combinations(range(X.shape[0]), 2)
        for i, j in iterator:
            out[i, j] = metric(X[i], Y[j], **kwds)

        # Make symmetric
        # NB: out += out.T will produce incorrect results
        out = out + out.T

        # Calculate diagonal
        # NB: nonzero diagonals are allowed for both metrics and kernels
        for i in range(X.shape[0]):
            x = X[i]
            out[i, i] = metric(x, x, **kwds)

    else:
        # Calculate all cells
        out = np.empty((X.shape[0], Y.shape[0]), dtype='float')
        iterator = itertools.product(range(X.shape[0]), range(Y.shape[0]))
        for i, j in iterator:
            out[i, j] = metric(X[i], Y[j], **kwds)

    return out

_VALID_METRICS = ['euclidean', 'l2', 'l1', 'manhattan', 'cityblock',
                  'braycurtis', 'canberra', 'chebyshev', 'correlation',
                  'cosine', 'dice', 'hamming', 'jaccard', 'kulsinski',
                  'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
                  'russellrao', 'seuclidean', 'sokalmichener',
                  'sokalsneath', 'sqeuclidean', 'yule', "wminkowski"]

def pairwise_distances(X, Y=None, metric="euclidean", n_jobs=1, **kwds):
    """ Compute the distance matrix from a vector array X and optional Y.

    This method takes either a vector array or a distance matrix, and returns
    a distance matrix. If the input is a vector array, the distances are
    computed. If the input is a distances matrix, it is returned instead.

    This method provides a safe way to take a distance matrix as input, while
    preserving compatibility with many other algorithms that take a vector
    array.

    If Y is given (default is None), then the returned matrix is the pairwise
    distance between the arrays from both X and Y.

    Valid values for metric are:

    - From scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
      'manhattan']. These metrics support sparse matrix inputs.

    - From scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
      'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski', 'mahalanobis',
      'matching', 'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean',
      'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule']
      See the documentation for scipy.spatial.distance for details on these
      metrics. These metrics do not support sparse matrix inputs.

    Note that in the case of 'cityblock', 'cosine' and 'euclidean' (which are
    valid scipy.spatial.distance metrics), the scikit-learn implementation
    will be used, which is faster and has support for sparse matrices (except
    for 'cityblock'). For a verbose description of the metrics from
    scikit-learn, see the __doc__ of the sklearn.pairwise.distance_metrics
    function.

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array [n_samples_a, n_samples_a] if metric == "precomputed", or, \
             [n_samples_a, n_features] otherwise
        Array of pairwise distances between samples, or a feature array.

    Y : array [n_samples_b, n_features], optional
        An optional second feature array. Only allowed if metric != "precomputed".

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        allowed by scipy.spatial.distance.pdist for its metric parameter, or
        a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
        If metric is "precomputed", X is assumed to be a distance matrix.
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them.

    n_jobs : int
        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    `**kwds` : optional keyword parameters
        Any further parameters are passed directly to the distance function.
        If using a scipy.spatial.distance metric, the parameters are still
        metric dependent. See the scipy docs for usage examples.

    Returns
    -------
    D : array [n_samples_a, n_samples_a] or [n_samples_a, n_samples_b]
        A distance matrix D such that D_{i, j} is the distance between the
        ith and jth vectors of the given matrix X, if Y is None.
        If Y is not None, then D_{i, j} is the distance between the ith array
        from X and the jth array from Y.

    """
    if (metric not in _VALID_METRICS and
            not callable(metric) and metric != "precomputed"):
        raise ValueError("Unknown metric %s. "
                         "Valid metrics are %s, or 'precomputed', or a "
                         "callable" % (metric, _VALID_METRICS))

    if metric == "precomputed":
        X, _ = check_pairwise_arrays(X, Y, precomputed=True)
        return X
    elif metric in PAIRWISE_DISTANCE_FUNCTIONS:
        func = PAIRWISE_DISTANCE_FUNCTIONS[metric]
    elif callable(metric):
        func = partial(_pairwise_callable, metric=metric, **kwds)
    else:
        if issparse(X) or issparse(Y):
            raise TypeError("scipy distance metrics do not"
                            " support sparse matrices.")
        X, Y = check_pairwise_arrays(X, Y)
        if n_jobs == 1 and X is Y:
            return distance.squareform(distance.pdist(X, metric=metric,
                                                      **kwds))
        func = partial(distance.cdist, metric=metric, **kwds)

    return _parallel_pairwise(X, Y, func, n_jobs, **kwds)

# Helper functions - distance
PAIRWISE_KERNEL_FUNCTIONS = {
    # If updating this dictionary, update the doc in both distance_metrics()
    # and also in pairwise_distances()!
    'additive_chi2': additive_chi2_kernel,
    'chi2': chi2_kernel,
    'linear': linear_kernel,
    'polynomial': polynomial_kernel,
    'poly': polynomial_kernel,
    'rbf': rbf_kernel,
    'laplacian': laplacian_kernel,
    'sigmoid': sigmoid_kernel,
    'cosine': cosine_similarity, }

def kernel_metrics():
    """ Valid metrics for pairwise_kernels

    This function simply returns the valid pairwise distance metrics.
    It exists, however, to allow for a verbose description of the mapping for
    each of the valid strings.

    The valid distance metrics, and the function they map to, are:
      ===============   ========================================
      metric            Function
      ===============   ========================================
      'additive_chi2'   sklearn.pairwise.additive_chi2_kernel
      'chi2'            sklearn.pairwise.chi2_kernel
      'linear'          sklearn.pairwise.linear_kernel
      'poly'            sklearn.pairwise.polynomial_kernel
      'polynomial'      sklearn.pairwise.polynomial_kernel
      'rbf'             sklearn.pairwise.rbf_kernel
      'laplacian'       sklearn.pairwise.laplacian_kernel
      'sigmoid'         sklearn.pairwise.sigmoid_kernel
      'cosine'          sklearn.pairwise.cosine_similarity
      ===============   ========================================

    Read more in the :ref:`User Guide <metrics>`.
    """
    return PAIRWISE_KERNEL_FUNCTIONS

KERNEL_PARAMS = {
    "additive_chi2": (),
    "chi2": (),
    "cosine": (),
    "exp_chi2": frozenset(["gamma"]),
    "linear": (),
    "poly": frozenset(["gamma", "degree", "coef0"]),
    "polynomial": frozenset(["gamma", "degree", "coef0"]),
    "rbf": frozenset(["gamma"]),
    "laplacian": frozenset(["gamma"]),
    "sigmoid": frozenset(["gamma", "coef0"]),
}

def pairwise_kernels(X, Y=None, metric="linear", filter_params=False,
                     n_jobs=1, **kwds):
    """Compute the kernel between arrays X and optional array Y.

    This method takes either a vector array or a kernel matrix, and returns
    a kernel matrix. If the input is a vector array, the kernels are
    computed. If the input is a kernel matrix, it is returned instead.

    This method provides a safe way to take a kernel matrix as input, while
    preserving compatibility with many other algorithms that take a vector
    array.

    If Y is given (default is None), then the returned matrix is the pairwise
    kernel between the arrays from both X and Y.

    Valid values for metric are::
        ['rbf', 'sigmoid', 'polynomial', 'poly', 'linear', 'cosine']

    Read more in the :ref:`User Guide <metrics>`.

    Parameters
    ----------
    X : array [n_samples_a, n_samples_a] if metric == "precomputed", or, \
             [n_samples_a, n_features] otherwise
        Array of pairwise kernels between samples, or a feature array.

    Y : array [n_samples_b, n_features]
        A second feature array only if X has shape [n_samples_a, n_features].

    metric : string, or callable
        The metric to use when calculating kernel between instances in a
        feature array. If metric is a string, it must be one of the metrics
        in pairwise.PAIRWISE_KERNEL_FUNCTIONS.
        If metric is "precomputed", X is assumed to be a kernel matrix.
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them.

    n_jobs : int
        The number of jobs to use for the computation. This works by breaking
        down the pairwise matrix into n_jobs even slices and computing them in
        parallel.

        If -1 all CPUs are used. If 1 is given, no parallel computing code is
        used at all, which is useful for debugging. For n_jobs below -1,
        (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one
        are used.

    filter_params: boolean
        Whether to filter invalid parameters or not.

    `**kwds` : optional keyword parameters
        Any further parameters are passed directly to the kernel function.

    Returns
    -------
    K : array [n_samples_a, n_samples_a] or [n_samples_a, n_samples_b]
        A kernel matrix K such that K_{i, j} is the kernel between the
        ith and jth vectors of the given matrix X, if Y is None.
        If Y is not None, then K_{i, j} is the kernel between the ith array
        from X and the jth array from Y.

    Notes
    -----
    If metric is 'precomputed', Y is ignored and X is returned.

    """
    # import GPKernel locally to prevent circular imports
    from ..gaussian_process.kernels import Kernel as GPKernel

    if metric == "precomputed":
        X, _ = check_pairwise_arrays(X, Y, precomputed=True)
        return X
    elif isinstance(metric, GPKernel):
        func = metric.__call__
    elif metric in PAIRWISE_KERNEL_FUNCTIONS:
        if filter_params:
            kwds = dict((k, kwds[k]) for k in kwds
                        if k in KERNEL_PARAMS[metric])
        func = PAIRWISE_KERNEL_FUNCTIONS[metric]
    elif callable(metric):
        func = partial(_pairwise_callable, metric=metric, **kwds)
    else:
        raise ValueError("Unknown kernel %r" % metric)

    return _parallel_pairwise(X, Y, func, n_jobs, **kwds)