"""
.. index:: multidimensional scaling (mds)

.. index::
   single: projection; multidimensional scaling (mds)

**********************************
Multidimensional scaling (``mds``)
**********************************

The functionality to perform multidimensional scaling
(http://en.wikipedia.org/wiki/Multidimensional_scaling).

The main class to perform multidimensional scaling is
:class:`Orange.projection.mds.MDS`

.. autoclass:: Orange.projection.mds.MDS
   :members:
   :exclude-members: Torgerson, get_distance, get_stress, calc_stress, run

   .. automethod:: calc_stress(stress_func=SgnRelStress)
   .. automethod:: run(iter, stress_func=SgnRelStress, eps=1e-3, progress_callback=None)

Stress functions
================

Stress functions that can be used for MDS have to be implemented as functions
or callable classes:

    .. method:: \ __call__(correct, current, weight=1.0)

       Compute the stress using the correct and the current distance value (the
       :obj:`Orange.projection.mds.MDS.distances` and
       :obj:`Orange.projection.mds.MDS.projected_distances` elements).

       :param correct: correct (actual) distance between elements, represented by
           the two points.
       :type correct: float

       :param current: current distance between the points in the MDS space.
       :type current: float

This module provides the following stress functions:

   * :obj:`SgnRelStress`
   * :obj:`KruskalStress`
   * :obj:`SammonStress`
   * :obj:`SgnSammonStress`

Examples
========

MDS Scatterplot
---------------

The following script computes the Euclidean distance between the data
instances and runs MDS. Final coordinates are plotted with matplotlib
(not included with orange, http://matplotlib.sourceforge.net/).

Example (:download:`mds-scatterplot.py <code/mds-scatterplot.py>`)

.. literalinclude:: code/mds-scatterplot.py
    :lines: 7-

The script produces a file *mds-scatterplot.py.png*. Color denotes
the class. Iris is a relatively simple data set with respect to
classification; to no surprise we see that MDS finds such instance
placement in 2D where instances of different classes are well separated.
Note that MDS has no knowledge of points' classes.

.. image:: files/mds-scatterplot.png

A more advanced example
-----------------------

The following script performs 10 steps of Smacof optimization before computing
the stress. This is suitable if you have a large dataset and want to save some
time.

Example (:download:`mds-advanced.py <code/mds-advanced.py>`)

.. literalinclude:: code/mds-advanced.py
    :lines: 7-

A few representative lines of the output are::

    <-0.633911848068, 0.112218663096> [5.1, 3.5, 1.4, 0.2, 'Iris-setosa']
    <-0.624193906784, -0.111143872142> [4.9, 3.0, 1.4, 0.2, 'Iris-setosa']
    ...
    <0.265250980854, 0.237793982029> [7.0, 3.2, 4.7, 1.4, 'Iris-versicolor']
    <0.208580598235, 0.116296850145> [6.4, 3.2, 4.5, 1.5, 'Iris-versicolor']
    ...
    <0.635814905167, 0.238721415401> [6.3, 3.3, 6.0, 2.5, 'Iris-virginica']
    <0.356859534979, -0.175976261497> [5.8, 2.7, 5.1, 1.9, 'Iris-virginica']
    ...

"""

import numpy
from numpy.linalg import svd

import Orange.core
from Orange import orangeom as orangemds
from Orange.utils import deprecated_keywords
from Orange.utils import deprecated_members

KruskalStress = orangemds.KruskalStress()
SammonStress = orangemds.SammonStress()
SgnSammonStress = orangemds.SgnSammonStress()
SgnRelStress = orangemds.SgnRelStress()

PointList = Orange.core.FloatListList
FloatListList = Orange.core.FloatListList

def _mycompare((a,aa),(b,bb)):
    if a == b:
        return 0
    if a < b:
        return -1
    else:
        return 1

class PivotMDS(object):
    def __init__(self, distances=None, pivots=50, dim=2, **kwargs):
        self.dst = numpy.array([m for m in distances])
        self.n = len(self.dst)

        if type(pivots) == type(1):
            self.k = pivots
            self.pivots = numpy.random.permutation(len(self.dst))[:pivots]
            #self.pivots.sort()
        elif type(pivots) == type([]):
            self.pivots = pivots
            #self.pivots.sort()
            self.k = len(self.pivots)
        else:
            raise AttributeError('pivots')

    def optimize(self):
#        # Classical MDS (Torgerson)
#        J = identity(self.n) - (1/float(self.n))
#        B = -1/2. * dot(dot(J, self.dst**2), J)
#        w,v = linalg.eig(B)
#        tmp = zip([float(val) for val in w], range(self.n))
#        tmp.sort()
#        w1, w2 = tmp[-1][0], tmp[-2][0]
#        v1, v2 = v[:, tmp[-1][1]], v[:, tmp[-2][1]]
#        return v1 * sqrt(w1), v2 * sqrt(w2)

        # Pivot MDS
        d = self.dst[[self.pivots]].T
        C = d**2
        # double-center d
        cavg = numpy.sum(d, axis=0)/(self.k+0.0)      # column sum
        ravg = numpy.sum(d, axis=1)/(self.n+0.0)    # row sum
        tavg = numpy.sum(cavg)/(self.n+0.0)   # total sum
        # TODO: optimize
        for i in xrange(self.n):
            for j in xrange(self.k):
                C[i,j] += -ravg[i] - cavg[j]

        C = -0.5 * (C + tavg)
        w,v = numpy.linalg.eig(numpy.dot(C.T, C))
        tmp = zip([float(val) for val in w], range(self.n))
        tmp.sort()
        w1, w2 = tmp[-1][0], tmp[-2][0]
        v1, v2 = v[:, tmp[-1][1]], v[:, tmp[-2][1]]
        x = numpy.dot(C, v1)
        y = numpy.dot(C, v2)
        return x, y

@deprecated_members(
    {"projectedDistances": "projected_distances",
     "originalDistances": "original_distances",
     "avgStress": "avg_stress",
     "progressCallback": "progress_callback",
     "getStress": "calc_stress",
     "get_stress": "calc_stress",
     "calcStress": "calc_stress",
     "getDistance": "calc_distance",
     "get_distance": "calc_distance",
     "calcDistance": "calc_distance",
     "Torgerson": "torgerson",
     "SMACOFstep": "smacof_step",
     "LSMT": "lsmt"})
class MDS(object):
    """
    Main class for performing multidimensional scaling.

    :param distances: original dissimilarity - a distance matrix to operate on.
    :type distances: :class:`Orange.misc.SymMatrix`

    :param dim: dimension of the projected space.
    :type dim: int

    :param points: an initial configuration of points (optional)
    :type points: :class:`Orange.core.FloatListList`

    An instance of MDS object has the following attributes and functions:

    .. attribute:: points

       Holds the current configuration of projected points in an
       :class:`Orange.core.FloatListList` object.

    .. attribute:: distances

       An :class:`Orange.misc.SymMatrix` containing the distances that we
       want to achieve (lsmt changes these).

    .. attribute:: projected_distances

       An :class:`Orange.misc.SymMatrix` containing the distances between
       projected points.

    .. attribute:: original_distances

       An :class:`Orange.misc.SymMatrix` containing the original distances
       between points.

    .. attribute:: stress

       An :class:`Orange.misc.SymMatrix` holding the stress.

    .. attribute:: dim

       An integer holding the dimension of the projected space.

    .. attribute:: n

       An integer holding the number of elements (points).

    .. attribute:: avg_stress

       A float holding the average stress in the :obj:`stress` matrix.

    .. attribute:: progress_callback

       A function that gets called after each optimization step in the
       :func:`run` method.

    """

    def __init__(self, distances=None, dim=2, **kwargs):
        self.mds=orangemds.MDS(distances, dim, **kwargs)
        self.original_distances=Orange.misc.SymMatrix([m for m in self.distances])

    def __getattr__(self, name):
        if name in ["points", "projected_distances", "distances" ,"stress",
                    "progress_callback", "n", "dim", "avg_stress"]:
            #print "rec:",name
            return self.__dict__["mds"].__dict__[name]
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        #print "setattr"
        if name=="points":
            for i in range(len(value)):
                for j in range(len(value[i])):
                    self.mds.points[i][j]=value[i][j]
            return

        if name in ["projected_distances", "distances" ,"stress",
                    "progress_callback"]:
            self.mds.__setattr__(name, value)
        else:
            self.__dict__[name]=value

    def __nonzero__(self):
        return True

    def smacof_step(self):
        """
        Perform a single iteration of a Smacof algorithm that optimizes
        :obj:`stress` and updates the :obj:`points`.
        """
        self.mds.SMACOFstep()

    def calc_distance(self):
        """
        Compute the distances between points and update the
        :obj:`projected_distances` matrix.

        """
        self.mds.get_distance()

    @deprecated_keywords({"stressFunc": "stress_func"})
    def calc_stress(self, stress_func=SgnRelStress):
        """
        Compute the stress between the current :obj:`projected_distances` and
        :obj:`distances` matrix using *stress_func* and update the
        :obj:`stress` matrix and :obj:`avgStress` accordingly.

        """
        self.mds.getStress(stress_func)

    @deprecated_keywords({"stressFunc": "stress_func"})
    def optimize(self, iter, stress_func=SgnRelStress, eps=1e-3,
                 progress_callback=None):
        self.mds.progress_callback=progress_callback
        self.mds.optimize(iter, stress_func, eps)

    @deprecated_keywords({"stressFunc": "stress_func"})
    def run(self, iter, stress_func=SgnRelStress, eps=1e-3,
            progress_callback=None):
        """
        Perform optimization until stopping conditions are met.
        Stopping conditions are:

           * optimization runs for *iter* iterations of smacof_step function, or
           * stress improvement (old stress minus new stress) is smaller than
             eps * old stress.

        :param iter: maximum number of optimization iterations.
        :type iter: int

        :param stress_func: stress function.
        """
        self.optimize(iter, stress_func, eps, progress_callback)

    def torgerson(self):
        """
        Run the Torgerson algorithm that computes an initial analytical
        solution of the problem.

        """
        # Torgerson's initial approximation
        O = numpy.array([m for m in self.distances])

##        #B = matrixmultiply(O,O)
##        # bug!? B = O**2
##        B = dot(O,O)
##        # double-center B
##        cavg = sum(B, axis=0)/(self.n+0.0)      # column sum
##        ravg = sum(B, axis=1)/(self.n+0.0)    # row sum
##        tavg = sum(cavg)/(self.n+0.0)   # total sum
##        # B[row][column]
##        for i in xrange(self.n):
##            for j in xrange(self.n):
##                B[i,j] += -cavg[j]-ravg[i]
##        B = -0.5*(B+tavg)

        # B = double-center O**2 !!!
        J = numpy.identity(self.n) - (1/numpy.float(self.n))
        B = -0.5 * numpy.dot(numpy.dot(J, O**2), J)

        # SVD-solve B = ULU'
        #(U,L,V) = singular_value_decomposition(B)
        (U,L,V)=svd(B)
        # X = U(L^0.5)
        # # self.X = matrixmultiply(U,identity(self.n)*sqrt(L))
        # X is n-dimensional, we take the two dimensions with the largest singular values
        idx = numpy.argsort(L)[-self.dim:].tolist()
        idx.reverse()

        Lt = numpy.take(L,idx)   # take those singular values
        Ut = numpy.take(U,idx,axis=1) # take those columns that are enabled
        Dt = numpy.identity(self.dim)*numpy.sqrt(Lt)  # make a diagonal matrix, with squarooted values
        self.points = Orange.core.FloatListList(numpy.dot(Ut,Dt))
        self.freshD = 0

#        D = identity(self.n)*sqrt(L)  # make a diagonal matrix, with squarooted values
#        X = matrixmultiply(U,D)
#        self.X = take(X,idx,1)

    # Kruskal's monotone transformation
    def lsmt(self):
        """
        Execute Kruskal monotone transformation.

        """
        # optimize the distance transformation
        # build vector o
        effect = 0
        self.getDistance()
        o = []
        for i in xrange(1,self.n):
            for j in xrange(i):
                o.append((self.original_distances[i,j],(i,j)))
        o.sort(_mycompare)
        # find the ties in o, and construct the d vector sorting in order within ties
        d = []
        td = []
        uv = [] # numbers of consecutively tied o values
        (i,j) = o[0][1]
        distnorm = self.projected_distances[i,j]*self.projected_distances[i,j]
        td = [self.projected_distances[i,j]] # fetch distance
        for l in xrange(1,len(o)):
            # copy now sorted distances in an array
            # but sort distances within a tied o
            (i,j) = o[l][1]
            cd = self.projected_distances[i,j]
            distnorm += self.projected_distances[i,j]*self.projected_distances[i,j]
            if o[l][0] != o[l-1][0]:
                # differing value, flush
                sum = reduce(lambda x,y:x+y,td)+0.0
                d.append([sum,len(td),sum/len(td),td])
                td = []
            td.append(cd)
        sum = reduce(lambda x,y:x+y,td)+0.0
        d.append([sum,len(td),sum/len(td),td])
        ####
        # keep merging non-monotonous areas in d
        monotony = 0
        while not monotony and len(d) > 1:
            monotony = 1
            pi = 0 # index
            n = 1  # n-areas
            nd = []
            r = d[0] # current area
            for i in range(1,len(d)):
                tr = d[i]
                if r[2]>=tr[2]:
                    monotony = 0
                    effect = 1
                    r[0] += tr[0]
                    r[1] += tr[1]
                    r[2] = tr[0]/tr[1]
                    r[3] += tr[3]
                else:
                    nd.append(r)
                    r = tr
            nd.append(r)
            d = nd
        # normalizing multiplier
        sum = 0.0
        for i in d:
            sum += i[2]*i[2]*i[1]
        f = numpy.sqrt(distnorm/numpy.max(sum,1e-6))
        # transform O
        k = 0
        for i in d:
            for j in range(i[1]):
                (ii,jj) = o[k][1]
                self.distances[ii,jj] = f*i[2]
                k += 1
        assert(len(o) == k)
        self.freshD = 0
        return effect


#!/usr/bin/env python

"""Matplotlib 3D plotting example

Demonstrates plotting with matplotlib.
"""

import sys

from sample import sample

from sympy import sin, Symbol
from sympy.external import import_module

def mplot3d(f, var1, var2, show=True):
    """
    Plot a 3d function using matplotlib/Tk.
    """

    import warnings
    warnings.filterwarnings("ignore", "Could not match \S")

    p = import_module('pylab')
    # Try newer version first
    p3 = import_module('mpl_toolkits.mplot3d',
        __import__kwargs={'fromlist': ['something']}) or import_module('matplotlib.axes3d')
    if not p or not p3:
        sys.exit("Matplotlib is required to use mplot3d.")

    x, y, z = sample(f, var1, var2)

    fig = p.figure()
    ax = p3.Axes3D(fig)

    # ax.plot_surface(x,y,z) #seems to be a bug in matplotlib
    ax.plot_wireframe(x, y, z)

    ax.set_xlabel('X')
    ax.set_ylabel('Y')
    ax.set_zlabel('Z')

    if show:
        p.show()

def main():
    x = Symbol('x')
    y = Symbol('y')

    mplot3d(x**2 - y**2, (x, -10.0, 10.0, 20), (y, -10.0, 10.0, 20))
    # mplot3d(x**2+y**2, (x, -10.0, 10.0, 20), (y, -10.0, 10.0, 20))
    # mplot3d(sin(x)+sin(y), (x, -3.14, 3.14, 10), (y, -3.14, 3.14, 10))

if __name__ == "__main__":
    main()


"""
=================================================
Compare evoked responses for different conditions
=================================================

In this example, an Epochs object for visual and auditory responses is created.
Both conditions are then accessed by their respective names to create a sensor
layout plot of the related evoked responses.

"""

# Authors: Denis Engemann <denis.engemann@gmail.com>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>

# License: BSD (3-clause)

import matplotlib.pyplot as plt
import mne

from mne.viz import plot_evoked_topo
from mne.datasets import sample

print(__doc__)

data_path = sample.data_path()

###############################################################################
# Set parameters
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'
event_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw-eve.fif'
tmin = -0.2
tmax = 0.5

#   Setup for reading the raw data
raw = mne.io.read_raw_fif(raw_fname)
events = mne.read_events(event_fname)

#   Set up amplitude-peak rejection values for MEG channels
reject = dict(grad=4000e-13, mag=4e-12)

# Create epochs including different events
event_id = {'audio/left': 1, 'audio/right': 2,
            'visual/left': 3, 'visual/right': 4}
epochs = mne.Epochs(raw, events, event_id, tmin, tmax,
                    picks='meg', baseline=(None, 0), reject=reject)

# Generate list of evoked objects from conditions names
evokeds = [epochs[name].average() for name in ('left', 'right')]

###############################################################################
# Show topography for two different conditions

colors = 'blue', 'red'
title = 'MNE sample data\nleft vs right (A/V combined)'

plot_evoked_topo(evokeds, color=colors, title=title, background_color='w')

plt.show()


# -*- coding: utf-8 -*-
#
# ADCpy documentation build configuration file, created by
# sphinx-quickstart on Tue Oct 07 11:54:34 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
          'matplotlib.sphinxext.mathmpl',
          'matplotlib.sphinxext.only_directives',
          'matplotlib.sphinxext.plot_directive',
          'matplotlib.sphinxext.ipython_directive',
          'sphinx.ext.intersphinx',
          'sphinx.ext.autodoc',
          'sphinx.ext.doctest','numpydoc',
          'sphinx.ext.autosummary']
          #'numpydoc']
          #'ipython_console_highlighting',
          #'inheritance_diagram',
          #'numpydoc']

autodoc_member_order = 'alphabetical'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'ADCPy'
copyright = u'2014, California Department of Water Resources'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = 'dwrsmall.gif'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
# This prevents the weird 2-index result if you use numpydoc
html_domain_indices = ['py-modindex']

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'ADCPydoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'ADCPy.tex', u'ADCPy Documentation',
   u'Benjamin Saenz, David Ralston, Rusty Holleman,\nEd Gross, Eli Ateljevich', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
latex_domain_indices = ['py-modindex']

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'adcpy', u'ADCpy Documentation',
     [u'Benjamin Saenz, David Ralston, Rusty Holleman, Ed Gross, Eli Ateljevich'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'ADCpy', u'ADCpy Documentation',
   u'Benjamin Saenz, David Ralston, Rusty Holleman, Ed Gross, Eli Ateljevich', 'ADCPy', 'Tools for ADCP analysis and visualization.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


from SimpleCV import Image, Camera, Display, Color
import pygame as pg
import numpy as np
from pylab import *
from mpl_toolkits.mplot3d import axes3d
from matplotlib.backends.backend_agg import FigureCanvasAgg
import cv2

bins = 8
#precompute
idxs = []
colors = []
offset = bins/2
skip = 255/bins
for x in range(0,bins):
    for y in range(0,bins):
        for z in range(0,bins):
            b = ((x*skip)+offset)/255.0
            g = ((y*skip)+offset)/255.0
            r = ((z*skip)+offset)/255.0
            idxs.append((x,y,z,(r,g,b)))

# plot points in 3D
cam = Camera()
disp = Display((800,600))
fig = figure()
fig.set_size_inches( (10,7) )

canvas = FigureCanvasAgg(fig)
azim = 0
while disp.isNotDone():
    ax = fig.gca(projection='3d')
    ax.set_xlabel('BLUE', color=(0,0,1) )
    ax.set_ylabel('GREEN',color=(0,1,0))
    ax.set_zlabel('RED',color=(1,0,0))
    # Get the color histogram
    img = cam.getImage().scale(0.3)
    rgb = img.getNumpyCv2()
    hist = cv2.calcHist([rgb],[0,1,2],None,[bins,bins,bins],[0,256,0,256,0,256])
    hist = hist/np.max(hist)
    # render everything
    [ ax.plot([x],[y],[z],'.',markersize=max(hist[x,y,z]*100,6),color=color) for x,y,z,color in idxs if(hist[x][y][z]>0) ]
    #[ ax.plot([x],[y],[z],'.',color=color) for x,y,z,color in idxs if(hist[x][y][z]>0) ]
    ax.set_xlim3d(0, bins-1)
    ax.set_ylim3d(0, bins-1)
    ax.set_zlim3d(0, bins-1)
    azim = (azim+0.5)%360
    ax.view_init(elev=35, azim=azim)
    ########### convert matplotlib to  SimpleCV image
    canvas.draw()
    renderer = canvas.get_renderer()
    raw_data = renderer.tostring_rgb()
    size = canvas.get_width_height()
    surf = pg.image.fromstring(raw_data, size, "RGB")
    figure = Image(surf)
    ############ All done
    figure = figure.floodFill((0,0), tolerance=5,color=Color.WHITE)
    result = figure.blit(img, pos=(20,20))
    result.save(disp)
    fig.clf()


#!/usr/bin/python
# axis3d.py, original mplot3d version by John Porter
# Created: 23 Sep 2005
# Parts rewritten by Reinier Heeres <reinier@heeres.eu>

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from matplotlib.externals import six

import math
import copy

from matplotlib import lines as mlines, axis as maxis, \
        patches as mpatches
from . import art3d
from . import proj3d

import numpy as np

def get_flip_min_max(coord, index, mins, maxs):
    if coord[index] == mins[index]:
        return maxs[index]
    else:
        return mins[index]

def move_from_center(coord, centers, deltas, axmask=(True, True, True)):
    '''Return a coordinate that is moved by "deltas" away from the center.'''
    coord = copy.copy(coord)
    #print coord, centers, deltas, axmask
    for i in range(3):
        if not axmask[i]:
            continue
        if coord[i] < centers[i]:
            coord[i] -= deltas[i]
        else:
            coord[i] += deltas[i]
    return coord

def tick_update_position(tick, tickxs, tickys, labelpos):
    '''Update tick line and label position and style.'''

    for (label, on) in ((tick.label1, tick.label1On), \
            (tick.label2, tick.label2On)):
        if on:
            label.set_position(labelpos)

    tick.tick1On, tick.tick2On = True, False
    tick.tick1line.set_linestyle('-')
    tick.tick1line.set_marker('')
    tick.tick1line.set_data(tickxs, tickys)
    tick.gridline.set_data(0, 0)

class Axis(maxis.XAxis):

    # These points from the unit cube make up the x, y and z-planes
    _PLANES = (
        (0, 3, 7, 4), (1, 2, 6, 5),     # yz planes
        (0, 1, 5, 4), (3, 2, 6, 7),     # xz planes
        (0, 1, 2, 3), (4, 5, 6, 7),     # xy planes
    )

    # Some properties for the axes
    _AXINFO = {
        'x': {'i': 0, 'tickdir': 1, 'juggled': (1, 0, 2),
            'color': (0.95, 0.95, 0.95, 0.5)},
        'y': {'i': 1, 'tickdir': 0, 'juggled': (0, 1, 2),
            'color': (0.90, 0.90, 0.90, 0.5)},
        'z': {'i': 2, 'tickdir': 0, 'juggled': (0, 2, 1),
            'color': (0.925, 0.925, 0.925, 0.5)},
    }

    def __init__(self, adir, v_intervalx, d_intervalx, axes, *args, **kwargs):
        # adir identifies which axes this is
        self.adir = adir
        # data and viewing intervals for this direction
        self.d_interval = d_intervalx
        self.v_interval = v_intervalx

        # This is a temporary member variable.
        # Do not depend on this existing in future releases!
        self._axinfo = self._AXINFO[adir].copy()
        self._axinfo.update({'label' : {'va': 'center',
                                        'ha': 'center'},
                             'tick' : {'inward_factor': 0.2,
                                       'outward_factor': 0.1},
                             'axisline': {'linewidth': 0.75,
                                          'color': (0, 0, 0, 1)},
                             'grid' : {'color': (0.9, 0.9, 0.9, 1),
                                       'linewidth': 1.0},
                            })

        maxis.XAxis.__init__(self, axes, *args, **kwargs)

        self.set_rotate_label(kwargs.get('rotate_label', None))

    def init3d(self):
        self.line = mlines.Line2D(xdata=(0, 0), ydata=(0, 0),
                            linewidth=self._axinfo['axisline']['linewidth'],
                            color=self._axinfo['axisline']['color'],
                            antialiased=True,
                           )

        # Store dummy data in Polygon object
        self.pane = mpatches.Polygon(np.array([[0,0], [0,1], [1,0], [0,0]]),
                                    closed=False,
                                    alpha=0.8,
                                    facecolor=(1,1,1,0),
                                    edgecolor=(1,1,1,0))
        self.set_pane_color(self._axinfo['color'])

        self.axes._set_artist_props(self.line)
        self.axes._set_artist_props(self.pane)
        self.gridlines = art3d.Line3DCollection([], )
        self.axes._set_artist_props(self.gridlines)
        self.axes._set_artist_props(self.label)
        self.axes._set_artist_props(self.offsetText)
        # Need to be able to place the label at the correct location
        self.label._transform = self.axes.transData
        self.offsetText._transform = self.axes.transData

    def get_tick_positions(self):
        majorLocs = self.major.locator()
        self.major.formatter.set_locs(majorLocs)
        majorLabels = [self.major.formatter(val, i) for i, val in enumerate(majorLocs)]
        return majorLabels, majorLocs

    def get_major_ticks(self, numticks=None):
        ticks = maxis.XAxis.get_major_ticks(self, numticks)
        for t in ticks:
            t.tick1line.set_transform(self.axes.transData)
            t.tick2line.set_transform(self.axes.transData)
            t.gridline.set_transform(self.axes.transData)
            t.label1.set_transform(self.axes.transData)
            t.label2.set_transform(self.axes.transData)
        return ticks

    def set_pane_pos(self, xys):
        xys = np.asarray(xys)
        xys = xys[:,:2]
        self.pane.xy = xys
        self.stale = True

    def set_pane_color(self, color):
        '''Set pane color to a RGBA tuple'''
        self._axinfo['color'] = color
        self.pane.set_edgecolor(color)
        self.pane.set_facecolor(color)
        self.pane.set_alpha(color[-1])
        self.stale = True

    def set_rotate_label(self, val):
        '''
        Whether to rotate the axis label: True, False or None.
        If set to None the label will be rotated if longer than 4 chars.
        '''
        self._rotate_label = val
        self.stale = True

    def get_rotate_label(self, text):
        if self._rotate_label is not None:
            return self._rotate_label
        else:
            return len(text) > 4

    def _get_coord_info(self, renderer):
        minx, maxx, miny, maxy, minz, maxz = self.axes.get_w_lims()
        if minx > maxx:
            minx, maxx = maxx, minx
        if miny > maxy:
            miny, maxy = maxy, miny
        if minz > maxz:
            minz, maxz = maxz, minz
        mins = np.array((minx, miny, minz))
        maxs = np.array((maxx, maxy, maxz))
        centers = (maxs + mins) / 2.
        deltas = (maxs - mins) / 12.
        mins = mins - deltas / 4.
        maxs = maxs + deltas / 4.

        vals = mins[0], maxs[0], mins[1], maxs[1], mins[2], maxs[2]
        tc = self.axes.tunit_cube(vals, renderer.M)
        avgz = [tc[p1][2] + tc[p2][2] + tc[p3][2] + tc[p4][2] for \
                p1, p2, p3, p4 in self._PLANES]
        highs = np.array([avgz[2*i] < avgz[2*i+1] for i in range(3)])

        return mins, maxs, centers, deltas, tc, highs

    def draw_pane(self, renderer):
        renderer.open_group('pane3d')

        mins, maxs, centers, deltas, tc, highs = self._get_coord_info(renderer)

        info = self._axinfo
        index = info['i']
        if not highs[index]:
            plane = self._PLANES[2 * index]
        else:
            plane = self._PLANES[2 * index + 1]
        xys = [tc[p] for p in plane]
        self.set_pane_pos(xys)
        self.pane.draw(renderer)

        renderer.close_group('pane3d')

    def draw(self, renderer):
        self.label._transform = self.axes.transData
        renderer.open_group('axis3d')

        # code from XAxis
        majorTicks = self.get_major_ticks()
        majorLocs = self.major.locator()

        info = self._axinfo
        index = info['i']

        # filter locations here so that no extra grid lines are drawn
        locmin, locmax = self.get_view_interval()
        if locmin > locmax:
            locmin, locmax = locmax, locmin

        # Rudimentary clipping
        majorLocs = [loc for loc in majorLocs if
                     locmin <= loc <= locmax]
        self.major.formatter.set_locs(majorLocs)
        majorLabels = [self.major.formatter(val, i)
                       for i, val in enumerate(majorLocs)]

        mins, maxs, centers, deltas, tc, highs = self._get_coord_info(renderer)

        # Determine grid lines
        minmax = np.where(highs, maxs, mins)

        # Draw main axis line
        juggled = info['juggled']
        edgep1 = minmax.copy()
        edgep1[juggled[0]] = get_flip_min_max(edgep1, juggled[0], mins, maxs)

        edgep2 = edgep1.copy()
        edgep2[juggled[1]] = get_flip_min_max(edgep2, juggled[1], mins, maxs)
        pep = proj3d.proj_trans_points([edgep1, edgep2], renderer.M)
        centpt = proj3d.proj_transform(centers[0], centers[1], centers[2], renderer.M)
        self.line.set_data((pep[0][0], pep[0][1]), (pep[1][0], pep[1][1]))
        self.line.draw(renderer)

        # Grid points where the planes meet
        xyz0 = []
        for val in majorLocs:
            coord = minmax.copy()
            coord[index] = val
            xyz0.append(coord)

        # Draw labels
        peparray = np.asanyarray(pep)
        # The transAxes transform is used because the Text object
        # rotates the text relative to the display coordinate system.
        # Therefore, if we want the labels to remain parallel to the
        # axis regardless of the aspect ratio, we need to convert the
        # edge points of the plane to display coordinates and calculate
        # an angle from that.
        # TODO: Maybe Text objects should handle this themselves?
        dx, dy = (self.axes.transAxes.transform([peparray[0:2, 1]]) -
                  self.axes.transAxes.transform([peparray[0:2, 0]]))[0]

        lxyz = 0.5*(edgep1 + edgep2)

        # A rough estimate; points are ambiguous since 3D plots rotate
        ax_scale = self.axes.bbox.size / self.figure.bbox.size
        ax_inches = np.multiply(ax_scale, self.figure.get_size_inches())
        ax_points_estimate = sum(72. * ax_inches)
        deltas_per_point = 48. / ax_points_estimate
        default_offset = 21.
        labeldeltas = (self.labelpad + default_offset) * deltas_per_point\
            * deltas
        axmask = [True, True, True]
        axmask[index] = False
        lxyz = move_from_center(lxyz, centers, labeldeltas, axmask)
        tlx, tly, tlz = proj3d.proj_transform(lxyz[0], lxyz[1], lxyz[2], \
                renderer.M)
        self.label.set_position((tlx, tly))
        if self.get_rotate_label(self.label.get_text()):
            angle = art3d.norm_text_angle(math.degrees(math.atan2(dy, dx)))
            self.label.set_rotation(angle)
        self.label.set_va(info['label']['va'])
        self.label.set_ha(info['label']['ha'])
        self.label.draw(renderer)

        # Draw Offset text

        # Which of the two edge points do we want to
        # use for locating the offset text?
        if juggled[2] == 2 :
            outeredgep = edgep1
            outerindex = 0
        else :
            outeredgep = edgep2
            outerindex = 1

        pos = copy.copy(outeredgep)
        pos = move_from_center(pos, centers, labeldeltas, axmask)
        olx, oly, olz = proj3d.proj_transform(pos[0], pos[1], pos[2], renderer.M)
        self.offsetText.set_text( self.major.formatter.get_offset() )
        self.offsetText.set_position( (olx, oly) )
        angle = art3d.norm_text_angle(math.degrees(math.atan2(dy, dx)))
        self.offsetText.set_rotation(angle)
        # Must set rotation mode to "anchor" so that
        # the alignment point is used as the "fulcrum" for rotation.
        self.offsetText.set_rotation_mode('anchor')

        #-----------------------------------------------------------------------
        # Note: the following statement for determining the proper alignment of
        #       the offset text. This was determined entirely by trial-and-error
        #       and should not be in any way considered as "the way".  There are
        #       still some edge cases where alignment is not quite right, but
        #       this seems to be more of a geometry issue (in other words, I
        #       might be using the wrong reference points).
        #
        #   (TT, FF, TF, FT) are the shorthand for the tuple of
        #     (centpt[info['tickdir']] <= peparray[info['tickdir'], outerindex],
        #      centpt[index] <= peparray[index, outerindex])
        #
        #   Three-letters (e.g., TFT, FTT) are short-hand for the array
        #    of bools from the variable 'highs'.
        # ---------------------------------------------------------------------
        if centpt[info['tickdir']] > peparray[info['tickdir'], outerindex] :
            # if FT and if highs has an even number of Trues
            if (centpt[index] <= peparray[index, outerindex]
                and ((len(highs.nonzero()[0]) % 2) == 0)) :
                # Usually, this means align right, except for the FTT case,
                # in which offset for axis 1 and 2 are aligned left.
                if highs.tolist() == [False, True, True] and index in (1, 2) :
                    align = 'left'
                else :
                    align = 'right'
            else :
                # The FF case
                align = 'left'
        else :
            # if TF and if highs has an even number of Trues
            if (centpt[index] > peparray[index, outerindex]
                and ((len(highs.nonzero()[0]) % 2) == 0)) :
                # Usually mean align left, except if it is axis 2
                if index == 2 :
                    align = 'right'
                else :
                    align = 'left'
            else :
                # The TT case
                align = 'right'

        self.offsetText.set_va('center')
        self.offsetText.set_ha(align)
        self.offsetText.draw(renderer)

        # Draw grid lines
        if len(xyz0) > 0:
            # Grid points at end of one plane
            xyz1 = copy.deepcopy(xyz0)
            newindex = (index + 1) % 3
            newval = get_flip_min_max(xyz1[0], newindex, mins, maxs)
            for i in range(len(majorLocs)):
                xyz1[i][newindex] = newval

            # Grid points at end of the other plane
            xyz2 = copy.deepcopy(xyz0)
            newindex = (index + 2) %  3
            newval = get_flip_min_max(xyz2[0], newindex, mins, maxs)
            for i in range(len(majorLocs)):
                xyz2[i][newindex] = newval

            lines = list(zip(xyz1, xyz0, xyz2))
            if self.axes._draw_grid:
                self.gridlines.set_segments(lines)
                self.gridlines.set_color([info['grid']['color']] * len(lines))
                self.gridlines.draw(renderer, project=True)

        # Draw ticks
        tickdir = info['tickdir']
        tickdelta = deltas[tickdir]
        if highs[tickdir]:
            ticksign = 1
        else:
            ticksign = -1

        for tick, loc, label in zip(majorTicks, majorLocs, majorLabels):
            if tick is None:
                continue

            # Get tick line positions
            pos = copy.copy(edgep1)
            pos[index] = loc
            pos[tickdir] = edgep1[tickdir] + info['tick']['outward_factor'] * \
                                             ticksign * tickdelta
            x1, y1, z1 = proj3d.proj_transform(pos[0], pos[1], pos[2], \
                    renderer.M)
            pos[tickdir] = edgep1[tickdir] - info['tick']['inward_factor'] * \
                                             ticksign * tickdelta
            x2, y2, z2 = proj3d.proj_transform(pos[0], pos[1], pos[2], \
                    renderer.M)

            # Get position of label
            default_offset = 8.  # A rough estimate
            labeldeltas = (tick.get_pad() + default_offset) * deltas_per_point\
                * deltas

            axmask = [True, True, True]
            axmask[index] = False
            pos[tickdir] = edgep1[tickdir]
            pos = move_from_center(pos, centers, labeldeltas, axmask)
            lx, ly, lz = proj3d.proj_transform(pos[0], pos[1], pos[2], \
                    renderer.M)

            tick_update_position(tick, (x1, x2), (y1, y2), (lx, ly))
            tick.set_label1(label)
            tick.set_label2(label)
            tick.draw(renderer)

        renderer.close_group('axis3d')
        self.stale = False

    def get_view_interval(self):
        """return the Interval instance for this 3d axis view limits"""
        return self.v_interval

    def set_view_interval(self, vmin, vmax, ignore=False):
        if ignore:
            self.v_interval = vmin, vmax
        else:
            Vmin, Vmax = self.get_view_interval()
            self.v_interval = min(vmin, Vmin), max(vmax, Vmax)

    # TODO: Get this to work properly when mplot3d supports
    #       the transforms framework.
    def get_tightbbox(self, renderer) :
        # Currently returns None so that Axis.get_tightbbox
        # doesn't return junk info.
        return None

# Use classes to look at different data limits

class XAxis(Axis):
    def get_data_interval(self):
        'return the Interval instance for this axis data limits'
        return self.axes.xy_dataLim.intervalx

class YAxis(Axis):
    def get_data_interval(self):
        'return the Interval instance for this axis data limits'
        return self.axes.xy_dataLim.intervaly

class ZAxis(Axis):
    def get_data_interval(self):
        'return the Interval instance for this axis data limits'
        return self.axes.zz_dataLim.intervalx


# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import copy
import numpy as np
import os
import pandas
import shutil
import tempfile
import unittest

import nab.corpus
from nab.util import recur

class CorpusTest(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    depth = 3

    cls.root = recur(os.path.dirname, os.path.realpath(__file__), depth)
    cls.corpusSource = os.path.join(cls.root, "tests", "test_data")

  def setUp(self):
    self.corpus = nab.corpus.Corpus(self.corpusSource)

  def testGetDataFiles(self):
    """
    Test the getDataFiles() function, specifically check if corpus.dataFiles
    is a dictionary containing DataFile objects containing pandas.DataFrame
    objects to represent the underlying data.
    """
    for df in self.corpus.dataFiles.values():
      self.assertIsInstance(df, nab.corpus.DataFile)
      self.assertIsInstance(df.data, pandas.DataFrame)
      self.assertEqual(set(df.data.columns.values),
        set(["timestamp", "value"]))

  def testAddColumn(self):
    """
    Test the addColumn() function, specificially check if a new column named
    "test" is added.
    """
    columnData = {}
    for relativePath, df in self.corpus.dataFiles.iteritems():
      rows, _ = df.data.shape
      columnData[relativePath] = pandas.Series(np.zeros(rows))

    self.corpus.addColumn("test", columnData, write=False)

    for df in self.corpus.dataFiles.values():
      self.assertEqual(set(df.data.columns.values),
        set(["timestamp", "value", "test"]))

  def testRemoveColumn(self):
    """
    Test the removeColumn() function, specifically check if an added column
    named "test" is removed.
    """
    columnData = {}
    for relativePath, df in self.corpus.dataFiles.iteritems():
      rows, _ = df.data.shape
      columnData[relativePath] = pandas.Series(np.zeros(rows))

    self.corpus.addColumn("test", columnData, write=False)

    self.corpus.removeColumn("test", write=False)

    for df in self.corpus.dataFiles.values():
      self.assertEqual(set(df.data.columns.values),
        set(["timestamp", "value"]))

  def testCopy(self):
    """
    Test the copy() function, specifically check if it copies the whole corpus
    to another directory and that the copied corpus is the exact same as the
    original.
    """
    copyLocation = os.path.join(tempfile.mkdtemp(), "test")
    self.corpus.copy(copyLocation)

    copyCorpus = nab.corpus.Corpus(copyLocation)

    for relativePath in self.corpus.dataFiles.keys():
      self.assertIn(relativePath, copyCorpus.dataFiles.keys())

      self.assertTrue(
        all(self.corpus.dataFiles[relativePath].data == \
            copyCorpus.dataFiles[relativePath].data))

    shutil.rmtree(copyLocation)

  def testAddDataSet(self):
    """
    Test the addDataSet() function, specifically check if it adds a new
    data file in the correct location in directory and into the dataFiles
    attribute.
    """
    copyLocation = os.path.join(tempfile.mkdtemp(), "test")
    copyCorpus = self.corpus.copy(copyLocation)

    for relativePath, df in self.corpus.dataFiles.iteritems():
      newPath = relativePath + "_copy"
      copyCorpus.addDataSet(newPath, copy.deepcopy(df))

      self.assertTrue(all(copyCorpus.dataFiles[newPath].data == df.data))

    shutil.rmtree(copyLocation)

  def testGetDataSubset(self):
    """
    Test the getDataSubset() function, specifically check if it returns only
    dataFiles with relativePaths that contain the query given.
    """
    query1 = "realAWSCloudwatch"
    subset1 = self.corpus.getDataSubset(query1)

    self.assertEqual(len(subset1), 2)
    for relativePath in subset1.keys():
      self.assertIn(query1, relativePath)

    query2 = "artificialWithAnomaly"
    subset2 = self.corpus.getDataSubset(query2)

    self.assertEqual(len(subset2), 1)

    for relativePath in subset2.keys():
      self.assertIn(query2, relativePath)

if __name__ == '__main__':
  unittest.main()


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 24 12:30:06 2016

@author: xf05id1
"""

from bluesky.callbacks import CallbackBase,LivePlot

#import os
#import time as ttime
#from databroker import DataBroker as db, get_events
#from databroker.databroker import fill_event
import filestore.api as fsapi
#from metadatastore.commands import run_start_given_uid, descriptors_by_start
#import matplotlib.pyplot as plt
from xray_vision.backend.mpl.cross_section_2d import CrossSection
#from .callbacks import CallbackBase
#import numpy as np
#import doct
#from databroker import DataBroker as db

i0_baseline = 7.24e-10

class NormalizeLivePlot(LivePlot):
    def __init__(self, *args, norm_key=None,  **kwargs):
        super().__init__(*args, **kwargs)
        if norm_key is None:
            raise RuntimeError("norm key is required kwarg")
        self._norm = norm_key

    def event(self, doc):
        "Update line with data from this Event."
        try:
            if self.x is not None:
                # this try/except block is needed because multiple event streams
                # will be emitted by the RunEngine and not all event streams will
                # have the keys we want
                new_x = doc['data'][self.x]
            else:
                new_x = doc['seq_num']
            new_y = doc['data'][self.y]
            new_norm = doc['data'][self._norm]
        except KeyError:
            # wrong event stream, skip it
            return
        self.y_data.append(new_y / abs(new_norm-i0_baseline))
        self.x_data.append(new_x)
        self.current_line.set_data(self.x_data, self.y_data)
        # Rescale and redraw.
        self.ax.relim(visible_only=True)
        self.ax.autoscale_view(tight=True)
        self.ax.figure.canvas.draw_idle()

#class LiveImagePiXi(CallbackBase):
    """
    Stream 2D images in a cross-section viewer.
    Parameters
    ----------
    field : string
        name of data field in an Event
    Note
    ----
    Requires a matplotlib fix that is not released as of this writing. The
    relevant commit is a951b7.
    """
#    def __init__(self, field):
#        super().__init__()
#        self.field = field
#        fig = plt.figure()
#        self.cs = CrossSection(fig)
#        self.cs._fig.show()

#    def event(self, doc):
#        #uid = doc['data'][self.field]
#        #data = fsapi.retrieve(uid)
#        data = doc['data']['pixi_image']
#        self.cs.update_image(data)
#        self.cs._fig.canvas.draw()
#        self.cs._fig.canvas.flush_events()
#
def make_live_image(image_axes, key):
    """
    Example
    p--------

    fig, ax = plt.subplots()
    image_axes = ax.imshow(np.zeros((476, 512)), vmin=0, vmax=2)
    cb = make_live_image(image_axes, 'pixi_image_array_data')
    RE(Count([pixi]), subs={'event': [cb]})
    """
    def live_image(name, doc):
        if name != 'event':
            return
        image_axes.set_data(doc['data'][key].reshape(476, 512))
    return live_image

class SRXLiveImage(CallbackBase):
    """
    Stream 2D images in a cross-section viewer.

    Parameters
    ----------
    field : string name of data field in an Event

    Note
    ----
    Requires a matplotlib fix that is not released as of this writing. The
    relevant commit is a951b7.
    """
    def __init__(self, field):
        super().__init__()
        self.field = field
        fig = plt.figure()
        self.cs = CrossSection(fig)
        self.cs._fig.show()

    def event(self, doc):
        uid = doc['data'][self.field]
        data = fsapi.retrieve(uid)
        self.cs.update_image(data)
        self.cs._fig.canvas.draw_idle()


#
# Leon Adams
#
# Python Module for running a hopfield network to relocate the memory from a perturbed image.
# The raw data set is represented in png image format. This code takes the three color channels (rgb)
# Converts to a single channel gray scaled image and then transforms the output to a [-1,1] vector
# for use in calculation of a hobfield neural network.
#
# Dependencies: numpy; matplotlib
#
# Usage
# Can use as normal python module or can be used as a python script.
# When calling from command line as script supply corruption percent at end of call
#
# Example: python hopfield.py 2 3 4
# This will produced 2, 3, and 4 percent perturbation on the image file and then
# attempt to locate closest memorized pattern using hopfield network with hebb learning rule.
# If called without perturbation parameters default to [1, 5, 10, 15, 20, 25] corruption percentages.

# Output: output of the execution is a series of images showing first the perturbed
# image with the corrupted percentages in the title. Then we show the closest memorized
# image found from the hobfield network.

# begin import needed libraries
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# end import libraries

def rgb_to_gray_array(rgb):
    '''
    Helper function to convert from rgb tensor to matrix gray-scaled image representation.
    Input: rgb tensor matrix of the three rgb color channels.
    output: numpy array of gray-scaled numeric values.
    '''
    return np.dot(rgb[...,:3], np.array([0.299, 0.587, 0.114]))

def read_images(filenames):
    '''
    Read images to set to memory. Convert from rgb tensor to gray scale representation.
    Takes a list of filenames in directory containing pixel images. Returns a list
    of numpy arrays converted to gray-scale.
    '''
    data = [( mpimg.imread(number) ) for number in filenames]
    return data, data[0].shape

def create_vector_image(data_array):
    '''
    Converts a gray-scaled image to [-1, +1] vector representation for hopfield networks.
    '''
    data_array = np.where(data_array < 0.99, -1, 1)
    return data_array.flatten()

def print_unique_cnts(array):
    print( np.unique(array, return_counts=True ) )

def train(memories):
    '''
    Training function for hobfield neural network. Trained with Hebb update rule.
    '''
    rate, c = memories.shape
    Weight = np.zeros((c, c))
    for p in memories:
        Weight = Weight + np.outer(p,p)

    Weight[np.diag_indices(c)] = 0
    return Weight/rate

def look_up(Weight_matrix, candidate_pattern, shape, percent_corrupted, steps=5):
    '''
    Given a candidate pattern, lookup closet memorized stable state. Return the
    stable memorized state.
    '''
    sgn = np.vectorize(lambda x: -1 if x<0 else 1)

    img = None
    for i in range(steps):
        im = show_pattern(candidate_pattern, shape)
        candidate_pattern = sgn(np.dot(candidate_pattern, Weight_matrix))
        if img is None:
            img = plt.imshow(im, cmap=plt.cm.binary, interpolation='nearest')
            plt.title(str(percent_corrupted) + ' percent corrupted pixels')
        else:
            img.set_data(im)
        plt.pause(.2)
        plt.draw()

    return candidate_pattern

def hopfield_energy(Weight, patterns):
    '''
    Calculates the current energy value for a given pattern and weight matrix.
    '''
    return np.array([-0.5*np.dot(np.dot(p.T, Weight), p) for p in patterns])

def show_img(image, shape):
    '''
    Helper function to produce visualization of an image.
    '''
    plt.imshow(image.reshape(shape), cmap=plt.cm.binary, interpolation='nearest')
    plt.show()

def show_pattern(pattern, shape):
    return np.where(pattern < 0, 0, 1).reshape(shape)

def corrupts(pattern, percentage):
    '''
    Helper function for deriving corrupted pattern images. Specify stable memory pattern
    and the percentage of pixels to switch.
    '''

    counts = int( 2*np.ceil( len(pattern) * percentage / 200 ) )
    neg_mask = np.where(pattern <= 0)[0]
    pos_mask = np.where(pattern > 0)[0]

    neg_corrupt_indices = np.random.choice(neg_mask, counts/2, replace = False)
    pos_corrupt_indices = np.random.choice(pos_mask, counts/2, replace = False)

    corrupt_pattern = np.copy(pattern)
    corrupt_pattern[neg_corrupt_indices] = 1
    corrupt_pattern[pos_corrupt_indices] = -1
    return corrupt_pattern

data, shape = read_images(['datasets/C.png', 'datasets/D.png', 'datasets/J.png'])

stable_memories = np.array([create_vector_image(rgb_to_gray_array(array)) for array in data ])
norm_weight_matrix = train(stable_memories)

def test_stable_memories(stable_memory_patterns, corrupt_perentages):
    for memory in stable_memory_patterns:
        for percent in corrupt_perentages:
            crpt_memory = corrupts(memory, percent)
            look_up(norm_weight_matrix, crpt_memory, shape[0:2], percent_corrupted = percent, steps=5)

if __name__ == "__main__":
    user_input = sys.argv

    if len(user_input) > 1:
        test_stable_memories(stable_memories, [float(i) for i in user_input[1:] ])
    else:
        test_stable_memories(stable_memories, [1, 5, 10, 15, 20, 25])


#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
A simple example demonstrating Arrow in Spark.
Run with:
  ./bin/spark-submit examples/src/main/python/sql/arrow.py
"""

from __future__ import print_function

from pyspark.sql import SparkSession
from pyspark.sql.utils import require_minimum_pandas_version, require_minimum_pyarrow_version

require_minimum_pandas_version()
require_minimum_pyarrow_version()

def dataframe_with_arrow_example(spark):
    # $example on:dataframe_with_arrow$
    import numpy as np
    import pandas as pd

    # Enable Arrow-based columnar data transfers
    spark.conf.set("spark.sql.execution.arrow.enabled", "true")

    # Generate a Pandas DataFrame
    pdf = pd.DataFrame(np.random.rand(100, 3))

    # Create a Spark DataFrame from a Pandas DataFrame using Arrow
    df = spark.createDataFrame(pdf)

    # Convert the Spark DataFrame back to a Pandas DataFrame using Arrow
    result_pdf = df.select("*").toPandas()
    # $example off:dataframe_with_arrow$
    print("Pandas DataFrame result statistics:\n%s\n" % str(result_pdf.describe()))

def scalar_pandas_udf_example(spark):
    # $example on:scalar_pandas_udf$
    import pandas as pd

    from pyspark.sql.functions import col, pandas_udf
    from pyspark.sql.types import LongType

    # Declare the function and create the UDF
    def multiply_func(a, b):
        return a * b

    multiply = pandas_udf(multiply_func, returnType=LongType())

    # The function for a pandas_udf should be able to execute with local Pandas data
    x = pd.Series([1, 2, 3])
    print(multiply_func(x, x))
    # 0    1
    # 1    4
    # 2    9
    # dtype: int64

    # Create a Spark DataFrame, 'spark' is an existing SparkSession
    df = spark.createDataFrame(pd.DataFrame(x, columns=["x"]))

    # Execute function as a Spark vectorized UDF
    df.select(multiply(col("x"), col("x"))).show()
    # +-------------------+
    # |multiply_func(x, x)|
    # +-------------------+
    # |                  1|
    # |                  4|
    # |                  9|
    # +-------------------+
    # $example off:scalar_pandas_udf$

def grouped_map_pandas_udf_example(spark):
    # $example on:grouped_map_pandas_udf$
    from pyspark.sql.functions import pandas_udf, PandasUDFType

    df = spark.createDataFrame(
        [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
        ("id", "v"))

    @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP)
    def substract_mean(pdf):
        # pdf is a pandas.DataFrame
        v = pdf.v
        return pdf.assign(v=v - v.mean())

    df.groupby("id").apply(substract_mean).show()
    # +---+----+
    # | id|   v|
    # +---+----+
    # |  1|-0.5|
    # |  1| 0.5|
    # |  2|-3.0|
    # |  2|-1.0|
    # |  2| 4.0|
    # +---+----+
    # $example off:grouped_map_pandas_udf$

if __name__ == "__main__":
    spark = SparkSession \
        .builder \
        .appName("Python Arrow-in-Spark example") \
        .getOrCreate()

    print("Running Pandas to/from conversion example")
    dataframe_with_arrow_example(spark)
    print("Running pandas_udf scalar example")
    scalar_pandas_udf_example(spark)
    print("Running pandas_udf grouped map example")
    grouped_map_pandas_udf_example(spark)

    spark.stop()


#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""DNNRegressor with custom input_fn for Housing dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import itertools

import pandas as pd
import tensorflow as tf

tf.logging.set_verbosity(tf.logging.INFO)

COLUMNS = ["crim", "zn", "indus", "nox", "rm", "age",
           "dis", "tax", "ptratio", "medv"]
FEATURES = ["crim", "zn", "indus", "nox", "rm",
            "age", "dis", "tax", "ptratio"]
LABEL = "medv"

def get_input_fn(data_set, num_epochs=None, shuffle=True):
  return tf.estimator.inputs.pandas_input_fn(
      x=pd.DataFrame({k: data_set[k].values for k in FEATURES}),
      y=pd.Series(data_set[LABEL].values),
      num_epochs=num_epochs,
      shuffle=shuffle)

def main(unused_argv):
  # Load datasets
  training_set = pd.read_csv("boston_train.csv", skipinitialspace=True,
                             skiprows=1, names=COLUMNS)
  test_set = pd.read_csv("boston_test.csv", skipinitialspace=True,
                         skiprows=1, names=COLUMNS)

  # Set of 6 examples for which to predict median house values
  prediction_set = pd.read_csv("boston_predict.csv", skipinitialspace=True,
                               skiprows=1, names=COLUMNS)

  # Feature cols
  feature_cols = [tf.feature_column.numeric_column(k) for k in FEATURES]

  # Build 2 layer fully connected DNN with 10, 10 units respectively.
  regressor = tf.estimator.DNNRegressor(feature_columns=feature_cols,
                                        hidden_units=[10, 10],
                                        model_dir="/tmp/boston_model")

  # Train
  regressor.train(input_fn=get_input_fn(training_set), steps=5000)

  # Evaluate loss over one epoch of test_set.
  ev = regressor.evaluate(
      input_fn=get_input_fn(test_set, num_epochs=1, shuffle=False))
  loss_score = ev["loss"]
  print("Loss: {0:f}".format(loss_score))

  # Print out predictions over a slice of prediction_set.
  y = regressor.predict(
      input_fn=get_input_fn(prediction_set, num_epochs=1, shuffle=False))
  # .predict() returns an iterator of dicts; convert to a list and print
  # predictions
  predictions = list(p["predictions"] for p in itertools.islice(y, 6))
  print("Predictions: {}".format(str(predictions)))

if __name__ == "__main__":
  tf.app.run()


import sys
import csv
import sip
#import org.apache.log4j.{Level, Logger}
import matplotlib
#matplotlib.user('agg')
import matplotlib.pyplot as plt
plt.switch_backend('agg')
from matplotlib import rcParams
rcParams.update({'figure.autolayout': True})
from pyspark import SparkContext, SparkConf
from datetime import datetime
from operator import add, itemgetter
from collections import namedtuple
from datetime import datetime
import os
import time
from StringIO import StringIO

#Defining the fields, Creating a Flights class with the following fields as a tuple
#Each row is converted into a list

timestarted = time.time()

fields   = ('date', 'airline', 'flightnum', 'origin', 'dest', 'dep',
            'dep_delay', 'arv', 'arv_delay', 'airtime', 'distance')

Flight   = namedtuple('Flight', fields, verbose=True)

DATE_FMT = "%Y-%m-%d"

TIME_FMT = "%H%M"

# User Defined Functions

def toCSVLine(data):
  return ','.join(str(d) for d in data)

def split(line):
    reader = csv.reader(StringIO(line))
    return reader.next()

def parse(row):
    row[0]  = datetime.strptime(row[0], DATE_FMT).time()
    row[5]  = datetime.strptime(row[5], TIME_FMT).time()
    row[6]  = float(row[6])
    row[7]  = datetime.strptime(row[7], TIME_FMT).time()
    row[8]  = float(row[8])
    row[9]  = float(row[9])
    row[10] = float(row[10])
    return Flight(*row[:11])

def notHeader(row):
    return "Description" not in row

def plot(airlinesdelays):
    airlines = [d[0] for d in airlinesdelays]
    minutes  = [d[1] for d in airlinesdelays]
    index    = list(xrange(len(airlines)))
#Above we retrieved the respective columns from the list

#Here we mention the plot as a horizontal bar plot
    fig, axe = plt.subplots()
    bars = axe.barh(index, minutes)

    # Add the total minutes to the right
    for idx, air, min in zip(index, airlines, minutes):
        if min > 0:
            bars[idx].set_color('#d9230f')
            axe.annotate(" %0.0f min" % min, xy=(min+1, idx+0.5), va='center')
        else:
            bars[idx].set_color('#469408')
            axe.annotate(" %0.0f min" % min, xy=(10, idx+0.5), va='center')

    # Set the ticks
    ticks = plt.yticks([idx+ 0.5 for idx in index], airlines)
    xt = plt.xticks()[0]
    plt.xticks(xt, [' '] * len(xt))

    # minimize chart junk
    plt.grid(axis = 'x', color ='white', linestyle='-')

    plt.title('Total Minutes Delayed per Airline')
    plt.savefig('airlines.png')

#airlines.filter(notHeader).take(10)

#main method is the entry point for the following program

if __name__ == "__main__":

  conf = SparkConf().setAppName("average")
  sc = SparkContext(conf=conf)

  #setting log level to error
 # val rootLogger = Logger.getRootLogger()
 # rootLogger.setLevel(Level.ERROR)

  #importing data from HDFS for performing analysis
  airlines = sc.textFile(sys.argv[1])
 # airlines = sc.textFile("hdfs://192.168.1.8:8020/fltdata/airlines.csv")
  flights = sc.textFile(sys.argv[2])
  airports =sc.textFile(sys.argv[3])

  airlinesParsed = dict(airlines.map(split).collect())
  airportsParsed= airports.filter(notHeader).map(split)
 # print "without header and spliting up", airlines.take(10)
  # print "without header and spliting up", airlines.take(10)
  flightsParsed= flights.map(lambda x: x.split(',')).map(parse)
  #print "The average delay is "+str(sumCount[0]/float(sumCount[1]))
  airportDelays = flightsParsed.map(lambda x: (x.origin,x.dep_delay))
  # First find the total delay per airport
  airportTotalDelay=airportDelays.reduceByKey(lambda x,y:x+y)

  # Find the count per airport
  airportCount=airportDelays.mapValues(lambda x:1).reduceByKey(lambda x,y:x+y)

  # Join to have the sum, count in 1 RDD
  airportSumCount=airportTotalDelay.join(airportCount)
  # Compute avg delay per airport
  airportAvgDelay=airportSumCount.mapValues(lambda x : x[0]/float(x[1]))
  airportDelay = airportAvgDelay.sortBy(lambda x:-x[1])
  print "", airportDelay.take(10)
  airportLookup=airportsParsed.collectAsMap()

  #airlineLookup=airlinesParsed.collectAsMap()
  airline_lookup = sc.broadcast(airlinesParsed)
  airlinesdelays  = flightsParsed.map(lambda f: (airline_lookup.value[f.airline],add(f.dep_delay, f.arv_delay)))
  airlinesdelays  = delays.reduceByKey(add).collect()
  airlinesdelays  = sorted(delays, key=itemgetter(1))
  #tenairlines = delays.map(toCSVLine)
  ten = airportAvgDelay.map(lambda x: (airportLookup[x[0]],x[1]))
  #print "", ten.take(10)

 for d in airlinesdelays:
      print "%0.0f minutes delayed\t%s" % (d[1], d[0])
  airportBC=sc.broadcast(airportLookup)
  topTenAirportsWithDelays = airportAvgDelay.map(lambda x: (airportBC.value[x[0]],x[1])).sortBy(lambda x:-x[1])
  lines = topTenAirportsWithDelays.take(10)

  topten = "/home/hadoop/"
  tenairlines = "/home/hadoop/"

  #For collecting the outputs into csv files
  with open('topten', "w") as output:
      writer = csv.writer(output, lineterminator='\n')
      for val in lines:
          writer.writerows([val])
  with open('tenairlines',"w") as output:
      writer = csv.writer(output, lineterminator='\n')
      for val in delays:
          writer.writerows([val])

  plot(airlinesdelays)

  #Final time taken will be calculated here
  timetaken  = time.time()-timestarted
  print "", timetaken


import os
import codecs
import re

import jieba
import numpy as np
from tqdm import tqdm
from tensorflow.contrib import learn
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder

class TextData(object):
    def __init__(self,args):
        self.args = args
        corpus_dir = self.args.corpus_dir

        self.load_data(corpus_dir)

    def load_data(self,corpus_dir):
        self.text = []
        self.label = []
        self.label_name = {}
        self.max_doc_len = 0
        self.label_num = 0
        self.vocab_size = 0

        raw_text = []
        raw_label = []
        tag_list = os.listdir(corpus_dir)
        for tag in tqdm(tag_list,desc='load_data',leave=False):
            data_path = os.path.join(corpus_dir,tag)
            for data_file in os.listdir(data_path):
                file_name = os.path.join(data_path,data_file)
                with codecs.open(file_name,'r',encoding='utf-8') as fr_raw:
                    raw_content = fr_raw.read()
                    text_word = [word for word in jieba.cut(raw_content) if re.match(u".*[\u4e00-\u9fa5]+", word)]
                    if text_word.__len__() < self.args.max_doc_len:
                        raw_text.append(text_word)
                        raw_label.append(tag)

        labelEncode = LabelEncoder()
        num_label = labelEncode.fit_transform(raw_label)
        self.label = OneHotEncoder(sparse=False).fit_transform(np.reshape(num_label,[-1,1]))
        self.label_num = len(labelEncode.classes_)

        #self.max_doc_len = max([len(doc) for doc in raw_text])
        self.max_doc_len = self.args.max_doc_len
        vocab_processor = learn.preprocessing.VocabularyProcessor(self.max_doc_len,tokenizer_fn=tokenizer_fn)
        self.text = np.array(list(vocab_processor.fit_transform(raw_text)))
        self.vocab_size = len(vocab_processor.vocabulary_)

    def shuffle_data(self):
        np.random.seed(3)
        shuffled = np.random.permutation(np.arange(len(self.label)))
        self.text = self.text[shuffled]
        self.label = self.label[shuffled]

    def get_batches(self):
        self.shuffle_data()

        sample_size = len(self.text)
        batch_size = self.args.batch_size

        for i in range(0,sample_size,batch_size):
            yield self.text[i:min(i+batch_size,sample_size)],self.label[i:min(i+batch_size,sample_size)]

#chinese cut word has completed instead of tensorflow cut word that cannt support chinese word
def tokenizer_fn(iterator):
    for value in iterator:
        yield value


""" test code to make a design matrix
"""
import numpy as np
from nipy.neurospin.utils.design_matrix import dmtx_light

tr = 1.0
frametimes = np.linspace(0,127*tr,128)
conditions = [0,0,0,1,1,1,3,3,3]
onsets=[30,70,100,10,30,90,30,40,60]
hrf_model = 'Canonical'
motion = np.cumsum(np.random.randn(128,6),0)
add_reg_names = ['tx','ty','tz','rx','ry','rz']

#event-related design matrix
paradigm = np.vstack(([conditions, onsets])).T
x1,name1 = dmtx_light(frametimes, paradigm, drift_model='Polynomial',
                      drift_order=3, add_regs=motion, add_reg_names=add_reg_names)

# block design matrix
duration = 7*np.ones(9)
paradigm = np.vstack(([conditions, onsets, duration])).T
x2,name2 = dmtx_light(frametimes, paradigm, drift_model='Polynomial', drift_order=3)

# FIR model
paradigm = np.vstack(([conditions, onsets])).T
hrf_model = 'FIR'
x3,name3 = dmtx_light(frametimes, paradigm, hrf_model = 'FIR',
                      drift_model='Polynomial', drift_order=3,
                      fir_delays = range(1,6))

import matplotlib.pylab as mp
mp.figure()
mp.imshow(x1/np.sqrt(np.sum(x1**2,0)),interpolation='Nearest', aspect='auto')
mp.xlabel('conditions')
mp.ylabel('scan number')
if name1!=None:
    mp.xticks(np.arange(len(name1)),name1,rotation=60,ha='right')
    mp.subplots_adjust(top=0.95,bottom=0.25)
mp.title('Example of event-related design matrix')

mp.figure()
mp.imshow(x2/np.sqrt(np.sum(x2**2,0)),interpolation='Nearest', aspect='auto')
mp.xlabel('conditions')
mp.ylabel('scan number')
if name2!=None:
    mp.xticks(np.arange(len(name2)),name2,rotation=60,ha='right')
    mp.subplots_adjust(top=0.95,bottom=0.25)
mp.title('Example of block design matrix')

mp.figure()
mp.imshow(x3/np.sqrt(np.sum(x3**2,0)),interpolation='Nearest', aspect='auto')
mp.xlabel('conditions')
mp.ylabel('scan number')
if name3!=None:
    mp.xticks(np.arange(len(name3)),name3,rotation=60,ha='right')
    mp.subplots_adjust(top=0.95,bottom=0.25)
mp.title('Example of FIR design matrix')

mp.show()


"""
===================================================
Faces recognition example using eigenfaces and SVMs
===================================================

The dataset used in this example is a preprocessed excerpt of the
"Labeled Faces in the Wild", aka LFW_:

  http://vis-www.cs.umass.edu/lfw/lfw-funneled.tgz (233MB)

  .. _LFW: http://vis-www.cs.umass.edu/lfw/

  original source: http://scikit-learn.org/stable/auto_examples/applications/face_recognition.html

"""

print __doc__

from time import time
import logging
import pylab as pl
import numpy as np

from sklearn.cross_validation import train_test_split
from sklearn.datasets import fetch_lfw_people
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.decomposition import RandomizedPCA
from sklearn.svm import SVC

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')

###############################################################################
# Download the data, if not already on disk and load it as numpy arrays
lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4)

# introspect the images arrays to find the shapes (for plotting)
n_samples, h, w = lfw_people.images.shape
np.random.seed(42)

# for machine learning we use the data directly (as relative pixel
# position info is ignored by this model)
X = lfw_people.data
n_features = X.shape[1]

# the label to predict is the id of the person
y = lfw_people.target
target_names = lfw_people.target_names
n_classes = target_names.shape[0]

print "Total dataset size:"
print "n_samples: %d" % n_samples
print "n_features: %d" % n_features
print "n_classes: %d" % n_classes

###############################################################################
# Split into a training and testing set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)

###############################################################################
# Compute a PCA (eigenfaces) on the face dataset (treated as unlabeled
# dataset): unsupervised feature extraction / dimensionality reduction
n_components = 150

print "Extracting the top %d eigenfaces from %d faces" % (n_components, X_train.shape[0])
t0 = time()
pca = RandomizedPCA(n_components=n_components, whiten=True).fit(X_train)
print "done in %0.3fs" % (time() - t0)

eigenfaces = pca.components_.reshape((n_components, h, w))

print "Projecting the input data on the eigenfaces orthonormal basis"
t0 = time()
X_train_pca = pca.transform(X_train)
X_test_pca = pca.transform(X_test)
print "done in %0.3fs" % (time() - t0)

###############################################################################
# Train a SVM classification model

print "Fitting the classifier to the training set"
t0 = time()
param_grid = {
         'C': [1e3, 5e3, 1e4, 5e4, 1e5],
          'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1],
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf = GridSearchCV(SVC(kernel='rbf', class_weight='balanced'), param_grid)
clf = clf.fit(X_train_pca, y_train)
print "done in %0.3fs" % (time() - t0)
print "Best estimator found by grid search:"
print clf.best_estimator_

###############################################################################
# Quantitative evaluation of the model quality on the test set

print "Predicting the people names on the testing set"
t0 = time()
y_pred = clf.predict(X_test_pca)
print "done in %0.3fs" % (time() - t0)

print classification_report(y_test, y_pred, target_names=target_names)
print confusion_matrix(y_test, y_pred, labels=range(n_classes))

###############################################################################
# Qualitative evaluation of the predictions using matplotlib

def plot_gallery(images, titles, h, w, n_row=3, n_col=4):
    """Helper function to plot a gallery of portraits"""
    pl.figure(figsize=(1.8 * n_col, 2.4 * n_row))
    pl.subplots_adjust(bottom=0, left=.01, right=.99, top=.90, hspace=.35)
    for i in range(n_row * n_col):
        pl.subplot(n_row, n_col, i + 1)
        pl.imshow(images[i].reshape((h, w)), cmap=pl.cm.gray)
        pl.title(titles[i], size=12)
        pl.xticks(())
        pl.yticks(())

# plot the result of the prediction on a portion of the test set

def title(y_pred, y_test, target_names, i):
    pred_name = target_names[y_pred[i]].rsplit(' ', 1)[-1]
    true_name = target_names[y_test[i]].rsplit(' ', 1)[-1]
    return 'predicted: %s\ntrue:      %s' % (pred_name, true_name)

prediction_titles = [title(y_pred, y_test, target_names, i)
                         for i in range(y_pred.shape[0])]

plot_gallery(X_test, prediction_titles, h, w)

# plot the gallery of the most significative eigenfaces

eigenface_titles = ["eigenface %d" % i for i in range(eigenfaces.shape[0])]
plot_gallery(eigenfaces, eigenface_titles, h, w)

pl.show()


import os
import locale
import codecs
import nose

import numpy as np
from numpy import iinfo

import pandas as pd
from pandas import (date_range, Index, _np_version_under1p9)
import pandas.util.testing as tm
from pandas.tools.util import cartesian_product, to_numeric

CURRENT_LOCALE = locale.getlocale()
LOCALE_OVERRIDE = os.environ.get('LOCALE_OVERRIDE', None)

class TestCartesianProduct(tm.TestCase):

    def test_simple(self):
        x, y = list('ABC'), [1, 22]
        result1, result2 = cartesian_product([x, y])
        expected1 = np.array(['A', 'A', 'B', 'B', 'C', 'C'])
        expected2 = np.array([1, 22, 1, 22, 1, 22])
        tm.assert_numpy_array_equal(result1, expected1)
        tm.assert_numpy_array_equal(result2, expected2)

    def test_datetimeindex(self):
        # regression test for GitHub issue #6439
        # make sure that the ordering on datetimeindex is consistent
        x = date_range('2000-01-01', periods=2)
        result1, result2 = [Index(y).day for y in cartesian_product([x, x])]
        expected1 = np.array([1, 1, 2, 2], dtype=np.int32)
        expected2 = np.array([1, 2, 1, 2], dtype=np.int32)
        tm.assert_numpy_array_equal(result1, expected1)
        tm.assert_numpy_array_equal(result2, expected2)

    def test_empty(self):
        # product of empty factors
        X = [[], [0, 1], []]
        Y = [[], [], ['a', 'b', 'c']]
        for x, y in zip(X, Y):
            expected1 = np.array([], dtype=np.asarray(x).dtype)
            expected2 = np.array([], dtype=np.asarray(y).dtype)
            result1, result2 = cartesian_product([x, y])
            tm.assert_numpy_array_equal(result1, expected1)
            tm.assert_numpy_array_equal(result2, expected2)

        # empty product (empty input):
        result = cartesian_product([])
        expected = []
        tm.assert_equal(result, expected)

    def test_invalid_input(self):
        invalid_inputs = [1, [1], [1, 2], [[1], 2],
                          'a', ['a'], ['a', 'b'], [['a'], 'b']]
        msg = "Input must be a list-like of list-likes"
        for X in invalid_inputs:
            tm.assertRaisesRegexp(TypeError, msg, cartesian_product, X=X)

class TestLocaleUtils(tm.TestCase):

    @classmethod
    def setUpClass(cls):
        super(TestLocaleUtils, cls).setUpClass()
        cls.locales = tm.get_locales()

        if not cls.locales:
            raise nose.SkipTest("No locales found")

        tm._skip_if_windows()

    @classmethod
    def tearDownClass(cls):
        super(TestLocaleUtils, cls).tearDownClass()
        del cls.locales

    def test_get_locales(self):
        # all systems should have at least a single locale
        assert len(tm.get_locales()) > 0

    def test_get_locales_prefix(self):
        if len(self.locales) == 1:
            raise nose.SkipTest("Only a single locale found, no point in "
                                "trying to test filtering locale prefixes")
        first_locale = self.locales[0]
        assert len(tm.get_locales(prefix=first_locale[:2])) > 0

    def test_set_locale(self):
        if len(self.locales) == 1:
            raise nose.SkipTest("Only a single locale found, no point in "
                                "trying to test setting another locale")

        if LOCALE_OVERRIDE is None:
            lang, enc = 'it_CH', 'UTF-8'
        elif LOCALE_OVERRIDE == 'C':
            lang, enc = 'en_US', 'ascii'
        else:
            lang, enc = LOCALE_OVERRIDE.split('.')

        enc = codecs.lookup(enc).name
        new_locale = lang, enc

        if not tm._can_set_locale(new_locale):
            with tm.assertRaises(locale.Error):
                with tm.set_locale(new_locale):
                    pass
        else:
            with tm.set_locale(new_locale) as normalized_locale:
                new_lang, new_enc = normalized_locale.split('.')
                new_enc = codecs.lookup(enc).name
                normalized_locale = new_lang, new_enc
                self.assertEqual(normalized_locale, new_locale)

        current_locale = locale.getlocale()
        self.assertEqual(current_locale, CURRENT_LOCALE)

class TestToNumeric(tm.TestCase):

    def test_series(self):
        s = pd.Series(['1', '-3.14', '7'])
        res = to_numeric(s)
        expected = pd.Series([1, -3.14, 7])
        tm.assert_series_equal(res, expected)

        s = pd.Series(['1', '-3.14', 7])
        res = to_numeric(s)
        tm.assert_series_equal(res, expected)

    def test_series_numeric(self):
        s = pd.Series([1, 3, 4, 5], index=list('ABCD'), name='XXX')
        res = to_numeric(s)
        tm.assert_series_equal(res, s)

        s = pd.Series([1., 3., 4., 5.], index=list('ABCD'), name='XXX')
        res = to_numeric(s)
        tm.assert_series_equal(res, s)

        # bool is regarded as numeric
        s = pd.Series([True, False, True, True],
                      index=list('ABCD'), name='XXX')
        res = to_numeric(s)
        tm.assert_series_equal(res, s)

    def test_error(self):
        s = pd.Series([1, -3.14, 'apple'])
        msg = 'Unable to parse string "apple" at position 2'
        with tm.assertRaisesRegexp(ValueError, msg):
            to_numeric(s, errors='raise')

        res = to_numeric(s, errors='ignore')
        expected = pd.Series([1, -3.14, 'apple'])
        tm.assert_series_equal(res, expected)

        res = to_numeric(s, errors='coerce')
        expected = pd.Series([1, -3.14, np.nan])
        tm.assert_series_equal(res, expected)

        s = pd.Series(['orange', 1, -3.14, 'apple'])
        msg = 'Unable to parse string "orange" at position 0'
        with tm.assertRaisesRegexp(ValueError, msg):
            to_numeric(s, errors='raise')

    def test_error_seen_bool(self):
        s = pd.Series([True, False, 'apple'])
        msg = 'Unable to parse string "apple" at position 2'
        with tm.assertRaisesRegexp(ValueError, msg):
            to_numeric(s, errors='raise')

        res = to_numeric(s, errors='ignore')
        expected = pd.Series([True, False, 'apple'])
        tm.assert_series_equal(res, expected)

        # coerces to float
        res = to_numeric(s, errors='coerce')
        expected = pd.Series([1., 0., np.nan])
        tm.assert_series_equal(res, expected)

    def test_list(self):
        s = ['1', '-3.14', '7']
        res = to_numeric(s)
        expected = np.array([1, -3.14, 7])
        tm.assert_numpy_array_equal(res, expected)

    def test_list_numeric(self):
        s = [1, 3, 4, 5]
        res = to_numeric(s)
        tm.assert_numpy_array_equal(res, np.array(s, dtype=np.int64))

        s = [1., 3., 4., 5.]
        res = to_numeric(s)
        tm.assert_numpy_array_equal(res, np.array(s))

        # bool is regarded as numeric
        s = [True, False, True, True]
        res = to_numeric(s)
        tm.assert_numpy_array_equal(res, np.array(s))

    def test_numeric(self):
        s = pd.Series([1, -3.14, 7], dtype='O')
        res = to_numeric(s)
        expected = pd.Series([1, -3.14, 7])
        tm.assert_series_equal(res, expected)

        s = pd.Series([1, -3.14, 7])
        res = to_numeric(s)
        tm.assert_series_equal(res, expected)

    def test_all_nan(self):
        s = pd.Series(['a', 'b', 'c'])
        res = to_numeric(s, errors='coerce')
        expected = pd.Series([np.nan, np.nan, np.nan])
        tm.assert_series_equal(res, expected)

    def test_type_check(self):
        # GH 11776
        df = pd.DataFrame({'a': [1, -3.14, 7], 'b': ['4', '5', '6']})
        with tm.assertRaisesRegexp(TypeError, "1-d array"):
            to_numeric(df)
        for errors in ['ignore', 'raise', 'coerce']:
            with tm.assertRaisesRegexp(TypeError, "1-d array"):
                to_numeric(df, errors=errors)

    def test_scalar(self):
        self.assertEqual(pd.to_numeric(1), 1)
        self.assertEqual(pd.to_numeric(1.1), 1.1)

        self.assertEqual(pd.to_numeric('1'), 1)
        self.assertEqual(pd.to_numeric('1.1'), 1.1)

        with tm.assertRaises(ValueError):
            to_numeric('XX', errors='raise')

        self.assertEqual(to_numeric('XX', errors='ignore'), 'XX')
        self.assertTrue(np.isnan(to_numeric('XX', errors='coerce')))

    def test_numeric_dtypes(self):
        idx = pd.Index([1, 2, 3], name='xxx')
        res = pd.to_numeric(idx)
        tm.assert_index_equal(res, idx)

        res = pd.to_numeric(pd.Series(idx, name='xxx'))
        tm.assert_series_equal(res, pd.Series(idx, name='xxx'))

        res = pd.to_numeric(idx.values)
        tm.assert_numpy_array_equal(res, idx.values)

        idx = pd.Index([1., np.nan, 3., np.nan], name='xxx')
        res = pd.to_numeric(idx)
        tm.assert_index_equal(res, idx)

        res = pd.to_numeric(pd.Series(idx, name='xxx'))
        tm.assert_series_equal(res, pd.Series(idx, name='xxx'))

        res = pd.to_numeric(idx.values)
        tm.assert_numpy_array_equal(res, idx.values)

    def test_str(self):
        idx = pd.Index(['1', '2', '3'], name='xxx')
        exp = np.array([1, 2, 3], dtype='int64')
        res = pd.to_numeric(idx)
        tm.assert_index_equal(res, pd.Index(exp, name='xxx'))

        res = pd.to_numeric(pd.Series(idx, name='xxx'))
        tm.assert_series_equal(res, pd.Series(exp, name='xxx'))

        res = pd.to_numeric(idx.values)
        tm.assert_numpy_array_equal(res, exp)

        idx = pd.Index(['1.5', '2.7', '3.4'], name='xxx')
        exp = np.array([1.5, 2.7, 3.4])
        res = pd.to_numeric(idx)
        tm.assert_index_equal(res, pd.Index(exp, name='xxx'))

        res = pd.to_numeric(pd.Series(idx, name='xxx'))
        tm.assert_series_equal(res, pd.Series(exp, name='xxx'))

        res = pd.to_numeric(idx.values)
        tm.assert_numpy_array_equal(res, exp)

    def test_datetimelike(self):
        for tz in [None, 'US/Eastern', 'Asia/Tokyo']:
            idx = pd.date_range('20130101', periods=3, tz=tz, name='xxx')
            res = pd.to_numeric(idx)
            tm.assert_index_equal(res, pd.Index(idx.asi8, name='xxx'))

            res = pd.to_numeric(pd.Series(idx, name='xxx'))
            tm.assert_series_equal(res, pd.Series(idx.asi8, name='xxx'))

            res = pd.to_numeric(idx.values)
            tm.assert_numpy_array_equal(res, idx.asi8)

    def test_timedelta(self):
        idx = pd.timedelta_range('1 days', periods=3, freq='D', name='xxx')
        res = pd.to_numeric(idx)
        tm.assert_index_equal(res, pd.Index(idx.asi8, name='xxx'))

        res = pd.to_numeric(pd.Series(idx, name='xxx'))
        tm.assert_series_equal(res, pd.Series(idx.asi8, name='xxx'))

        res = pd.to_numeric(idx.values)
        tm.assert_numpy_array_equal(res, idx.asi8)

    def test_period(self):
        idx = pd.period_range('2011-01', periods=3, freq='M', name='xxx')
        res = pd.to_numeric(idx)
        tm.assert_index_equal(res, pd.Index(idx.asi8, name='xxx'))

        # ToDo: enable when we can support native PeriodDtype
        # res = pd.to_numeric(pd.Series(idx, name='xxx'))
        # tm.assert_series_equal(res, pd.Series(idx.asi8, name='xxx'))

    def test_non_hashable(self):
        # Test for Bug #13324
        s = pd.Series([[10.0, 2], 1.0, 'apple'])
        res = pd.to_numeric(s, errors='coerce')
        tm.assert_series_equal(res, pd.Series([np.nan, 1.0, np.nan]))

        res = pd.to_numeric(s, errors='ignore')
        tm.assert_series_equal(res, pd.Series([[10.0, 2], 1.0, 'apple']))

        with self.assertRaisesRegexp(TypeError, "Invalid object type"):
            pd.to_numeric(s)

    def test_downcast(self):
        # see gh-13352
        mixed_data = ['1', 2, 3]
        int_data = [1, 2, 3]
        date_data = np.array(['1970-01-02', '1970-01-03',
                              '1970-01-04'], dtype='datetime64[D]')

        invalid_downcast = 'unsigned-integer'
        msg = 'invalid downcasting method provided'

        smallest_int_dtype = np.dtype(np.typecodes['Integer'][0])
        smallest_uint_dtype = np.dtype(np.typecodes['UnsignedInteger'][0])

        # support below np.float32 is rare and far between
        float_32_char = np.dtype(np.float32).char
        smallest_float_dtype = float_32_char

        for data in (mixed_data, int_data, date_data):
            with self.assertRaisesRegexp(ValueError, msg):
                pd.to_numeric(data, downcast=invalid_downcast)

            expected = np.array([1, 2, 3], dtype=np.int64)

            res = pd.to_numeric(data)
            tm.assert_numpy_array_equal(res, expected)

            res = pd.to_numeric(data, downcast=None)
            tm.assert_numpy_array_equal(res, expected)

            expected = np.array([1, 2, 3], dtype=smallest_int_dtype)

            for signed_downcast in ('integer', 'signed'):
                res = pd.to_numeric(data, downcast=signed_downcast)
                tm.assert_numpy_array_equal(res, expected)

            expected = np.array([1, 2, 3], dtype=smallest_uint_dtype)
            res = pd.to_numeric(data, downcast='unsigned')
            tm.assert_numpy_array_equal(res, expected)

            expected = np.array([1, 2, 3], dtype=smallest_float_dtype)
            res = pd.to_numeric(data, downcast='float')
            tm.assert_numpy_array_equal(res, expected)

        # if we can't successfully cast the given
        # data to a numeric dtype, do not bother
        # with the downcast parameter
        data = ['foo', 2, 3]
        expected = np.array(data, dtype=object)
        res = pd.to_numeric(data, errors='ignore',
                            downcast='unsigned')
        tm.assert_numpy_array_equal(res, expected)

        # cannot cast to an unsigned integer because
        # we have a negative number
        data = ['-1', 2, 3]
        expected = np.array([-1, 2, 3], dtype=np.int64)
        res = pd.to_numeric(data, downcast='unsigned')
        tm.assert_numpy_array_equal(res, expected)

        # cannot cast to an integer (signed or unsigned)
        # because we have a float number
        data = ['1.1', 2, 3]
        expected = np.array([1.1, 2, 3], dtype=np.float64)

        for downcast in ('integer', 'signed', 'unsigned'):
            res = pd.to_numeric(data, downcast=downcast)
            tm.assert_numpy_array_equal(res, expected)

        # the smallest integer dtype need not be np.(u)int8
        data = ['256', 257, 258]

        for downcast, expected_dtype in zip(
                ['integer', 'signed', 'unsigned'],
                [np.int16, np.int16, np.uint16]):
            expected = np.array([256, 257, 258], dtype=expected_dtype)
            res = pd.to_numeric(data, downcast=downcast)
            tm.assert_numpy_array_equal(res, expected)

    def test_downcast_limits(self):
        # Test the limits of each downcast. Bug: #14401.
        # Check to make sure numpy is new enough to run this test.
        if _np_version_under1p9:
            raise nose.SkipTest("Numpy version is under 1.9")

        i = 'integer'
        u = 'unsigned'
        dtype_downcast_min_max = [
            ('int8', i, [iinfo(np.int8).min, iinfo(np.int8).max]),
            ('int16', i, [iinfo(np.int16).min, iinfo(np.int16).max]),
            ('int32', i, [iinfo(np.int32).min, iinfo(np.int32).max]),
            ('int64', i, [iinfo(np.int64).min, iinfo(np.int64).max]),
            ('uint8', u, [iinfo(np.uint8).min, iinfo(np.uint8).max]),
            ('uint16', u, [iinfo(np.uint16).min, iinfo(np.uint16).max]),
            ('uint32', u, [iinfo(np.uint32).min, iinfo(np.uint32).max]),
            # Test will be skipped until there is more uint64 support.
            # ('uint64', u, [iinfo(uint64).min, iinfo(uint64).max]),
            ('int16', i, [iinfo(np.int8).min, iinfo(np.int8).max + 1]),
            ('int32', i, [iinfo(np.int16).min, iinfo(np.int16).max + 1]),
            ('int64', i, [iinfo(np.int32).min, iinfo(np.int32).max + 1]),
            ('int16', i, [iinfo(np.int8).min - 1, iinfo(np.int16).max]),
            ('int32', i, [iinfo(np.int16).min - 1, iinfo(np.int32).max]),
            ('int64', i, [iinfo(np.int32).min - 1, iinfo(np.int64).max]),
            ('uint16', u, [iinfo(np.uint8).min, iinfo(np.uint8).max + 1]),
            ('uint32', u, [iinfo(np.uint16).min, iinfo(np.uint16).max + 1]),
            # Test will be skipped until there is more uint64 support.
            # ('uint64', u, [iinfo(np.uint32).min, iinfo(np.uint32).max + 1]),
        ]

        for dtype, downcast, min_max in dtype_downcast_min_max:
            series = pd.to_numeric(pd.Series(min_max), downcast=downcast)
            tm.assert_equal(series.dtype, dtype)

if __name__ == '__main__':
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)