#
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import division

import numpy as np
import pandas as pd
import re

from zipline.errors import IncompatibleHistoryFrequency

def parse_freq_str(freq_str):
    # TODO: Wish we were more aligned with pandas here.
    num_str, unit_str = re.match('([0-9]+)([A-Za-z]+)', freq_str).groups()
    return int(num_str), unit_str

class Frequency(object):
    """
    Represents how the data is sampled, as specified by the algoscript
    via units like "1d", "1m", etc.

    Currently only two frequencies are supported, "1d" and "1m"

    - "1d" provides data at daily frequency, with the latest bar aggregating
    the elapsed minutes of the (incomplete) current day
    - "1m" provides data at minute frequency
    """
    SUPPORTED_FREQUENCIES = frozenset({'1d', '1m'})
    MAX_MINUTES = {'m': 1, 'd': 390}
    MAX_DAYS = {'d': 1}

    def __init__(self, freq_str, data_frequency, env):

        if freq_str not in self.SUPPORTED_FREQUENCIES:
            raise ValueError(
                "history frequency must be in {supported}".format(
                    supported=self.SUPPORTED_FREQUENCIES,
                ))
        # The string the at the algoscript specifies.
        # Hold onto to use a key for caching.
        self.freq_str = freq_str

        # num - The number of units of the frequency.
        # unit_str - The unit type, e.g. 'd'
        self.num, self.unit_str = parse_freq_str(freq_str)

        self.data_frequency = data_frequency
        self.env = env

    def next_window_start(self, previous_window_close):
        """
        Get the first minute of the window starting after a window that
        finished on @previous_window_close.
        """
        if self.unit_str == 'd':
            return self.next_day_window_start(previous_window_close, self.env,
                                              self.data_frequency)
        elif self.unit_str == 'm':
            return self.env.next_market_minute(previous_window_close)

    @staticmethod
    def next_day_window_start(previous_window_close, env,
                              data_frequency='minute'):
        """
        Get the next day window start after @previous_window_close.  This is
        defined as the first market open strictly greater than
        @previous_window_close.
        """
        if data_frequency == 'daily':
            next_open = env.next_trading_day(previous_window_close)
        else:
            next_open = env.next_market_minute(previous_window_close)
        return next_open

    def window_open(self, window_close):
        """
        For a period ending on `window_end`, calculate the date of the first
        minute bar that should be used to roll a digest for this frequency.
        """
        if self.unit_str == 'd':
            return self.day_window_open(window_close, self.num)
        elif self.unit_str == 'm':
            return self.minute_window_open(window_close, self.num)

    def window_close(self, window_start):
        """
        For a period starting on `window_start`, calculate the date of the last
        minute bar that should be used to roll a digest for this frequency.
        """
        if self.unit_str == 'd':
            return self.day_window_close(window_start, self.num)
        elif self.unit_str == 'm':
            return self.minute_window_close(window_start, self.num)

    def day_window_open(self, window_close, num_days):
        """
        Get the first minute for a daily window of length @num_days with last
        minute @window_close.  This is calculated by searching backward until
        @num_days market_closes are encountered.
        """
        open_ = self.env.open_close_window(
            window_close,
            1,
            offset=-(num_days - 1)
        ).market_open.iloc[0]

        if self.data_frequency == 'daily':
            open_ = pd.tslib.normalize_date(open_)

        return open_

    def minute_window_open(self, window_close, num_minutes):
        """
        Get the first minute for a minutely window of length @num_minutes with
        last minute @window_close.

        This is defined as window_close if num_minutes == 1, and otherwise as
        the N-1st market minute after @window_start.
        """
        if num_minutes == 1:
            # Short circuit this case.
            return window_close

        return self.env.market_minute_window(
            window_close, count=-num_minutes
        )[-1]

    def day_window_close(self, window_start, num_days):
        """
        Get the window close for a daily frequency.
        If the data_frequency is minute, then this will be the last minute of
        last day of the window.

        If the data_frequency is minute, this will be midnight utc of the last
        day of the window.
        """
        if self.data_frequency != 'daily':
            return self.env.get_open_and_close(
                self.env.add_trading_days(num_days - 1, window_start),
            )[1]

        return pd.tslib.normalize_date(
            self.env.add_trading_days(num_days - 1, window_start),
        )

    def minute_window_close(self, window_start, num_minutes):
        """
        Get the last minute for a minutely window of length @num_minutes with
        first minute @window_start.

        This is defined as window_start if num_minutes == 1, and otherwise as
        the N-1st market minute after @window_start.
        """
        if num_minutes == 1:
            # Short circuit this case.
            return window_start

        return self.env.market_minute_window(
            window_start, count=num_minutes
        )[-1]

    def prev_bar(self, dt):
        """
        Returns the previous bar for dt.
        """
        if self.unit_str == 'd':
            if self.data_frequency == 'minute':
                def func(dt):
                    return self.env.get_open_and_close(
                        self.env.previous_trading_day(dt))[1]
            else:
                func = self.env.previous_trading_day
        else:
            func = self.env.previous_market_minute

        # Cache the function dispatch.
        self.prev_bar = func

        return func(dt)

    @property
    def max_bars(self):
        if self.data_frequency == 'daily':
            return self.max_days
        else:
            return self.max_minutes

    @property
    def max_days(self):
        if self.data_frequency != 'daily':
            raise ValueError('max_days requested in minute mode')
        return self.MAX_DAYS[self.unit_str] * self.num

    @property
    def max_minutes(self):
        """
        The maximum number of minutes required to roll a bar at this frequency.
        """
        if self.data_frequency != 'minute':
            raise ValueError('max_minutes requested in daily mode')
        return self.MAX_MINUTES[self.unit_str] * self.num

    def normalize(self, dt):
        if self.data_frequency != 'daily':
            return dt
        return pd.tslib.normalize_date(dt)

    def __eq__(self, other):
        return self.freq_str == other.freq_str

    def __hash__(self):
        return hash(self.freq_str)

    def __repr__(self):
        return ''.join([str(self.__class__.__name__),
                        "('", self.freq_str, "')"])

class HistorySpec(object):
    """
    Maps to the parameters of the history() call made by the algoscript

    An object is used here so that get_history calls are not constantly
    parsing the parameters and provides values for caching and indexing into
    result frames.
    """

    FORWARD_FILLABLE = frozenset({'price'})

    @classmethod
    def spec_key(cls, bar_count, freq_str, field, ffill):
        """
        Used as a hash/key value for the HistorySpec.
        """
        return "{0}:{1}:{2}:{3}".format(
            bar_count, freq_str, field, ffill)

    def __init__(self, bar_count, frequency, field, ffill, env,
                 data_frequency='daily'):

        # Number of bars to look back.
        self.bar_count = bar_count
        if isinstance(frequency, str):
            frequency = Frequency(frequency, data_frequency, env)
        if frequency.unit_str == 'm' and data_frequency == 'daily':
            raise IncompatibleHistoryFrequency(
                frequency=frequency.unit_str,
                data_frequency=data_frequency,
            )

        # The frequency at which the data is sampled.
        self.frequency = frequency
        # The field, e.g. 'price', 'volume', etc.
        self.field = field
        # Whether or not to forward fill nan data.  Only has an effect if this
        # spec's field is in FORWARD_FILLABLE.
        self._ffill = ffill

        # Calculate the cache key string once.
        self.key_str = self.spec_key(
            bar_count, frequency.freq_str, field, ffill)

    @property
    def ffill(self):
        """
        Wrapper around self._ffill that returns False for fields which are not
        forward-fillable.
        """
        return self._ffill and self.field in self.FORWARD_FILLABLE

    def __repr__(self):
        return ''.join([self.__class__.__name__, "('", self.key_str, "')"])

def days_index_at_dt(history_spec, algo_dt, env):
    """
    Get the index of a frame to be used for a get_history call with daily
    frequency.
    """
    # Get the previous (bar_count - 1) days' worth of market closes.
    day_delta = (history_spec.bar_count - 1) * history_spec.frequency.num
    market_closes = env.open_close_window(
        algo_dt,
        day_delta,
        offset=(-day_delta),
        step=history_spec.frequency.num,
    ).market_close

    if history_spec.frequency.data_frequency == 'daily':
        market_closes = market_closes.apply(pd.tslib.normalize_date)

    # Append the current algo_dt as the last index value.
    # Using the 'rawer' numpy array values here because of a bottleneck
    # that appeared when using DatetimeIndex
    return np.append(market_closes.values, algo_dt)

def minutes_index_at_dt(history_spec, algo_dt, env):
    """
    Get the index of a frame to be used for a get_history_call with minutely
    frequency.
    """
    # TODO: This is almost certainly going to be too slow for production.
    return env.market_minute_window(
        algo_dt,
        history_spec.bar_count,
        step=-1,
    )[::-1]

def index_at_dt(history_spec, algo_dt, env):
    """
    Returns index of a frame returned by get_history() with the given
    history_spec and algo_dt.

    The resulting index will have @history_spec.bar_count bars, increasing in
    units of @history_spec.frequency, terminating at the given @algo_dt.

    Note: The last bar of the returned frame represents an as-of-yet incomplete
    time window, so the delta between the last and second-to-last bars is
    usually always less than `@history_spec.frequency` for frequencies greater
    than 1m.
    """
    frequency = history_spec.frequency
    if frequency.unit_str == 'd':
        return days_index_at_dt(history_spec, algo_dt, env)
    elif frequency.unit_str == 'm':
        return minutes_index_at_dt(history_spec, algo_dt, env)


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import six

import os
import sys
from hashlib import md5

from docutils import nodes
from docutils.parsers.rst import directives
import warnings

from matplotlib import rcParams
from matplotlib.mathtext import MathTextParser
rcParams['mathtext.fontset'] = 'cm'
mathtext_parser = MathTextParser("Bitmap")

# Define LaTeX math node:
class latex_math(nodes.General, nodes.Element):
    pass

def fontset_choice(arg):
    return directives.choice(arg, ['cm', 'stix', 'stixsans'])

options_spec = {'fontset': fontset_choice}

def math_role(role, rawtext, text, lineno, inliner,
              options={}, content=[]):
    i = rawtext.find('`')
    latex = rawtext[i+1:-1]
    node = latex_math(rawtext)
    node['latex'] = latex
    node['fontset'] = options.get('fontset', 'cm')
    return [node], []
math_role.options = options_spec

def math_directive(name, arguments, options, content, lineno,
                   content_offset, block_text, state, state_machine):
    latex = ''.join(content)
    node = latex_math(block_text)
    node['latex'] = latex
    node['fontset'] = options.get('fontset', 'cm')
    return [node]

# This uses mathtext to render the expression
def latex2png(latex, filename, fontset='cm'):
    latex = "$%s$" % latex
    orig_fontset = rcParams['mathtext.fontset']
    rcParams['mathtext.fontset'] = fontset
    if os.path.exists(filename):
        depth = mathtext_parser.get_depth(latex, dpi=100)
    else:
        try:
            depth = mathtext_parser.to_png(filename, latex, dpi=100)
        except:
            warnings.warn("Could not render math expression %s" % latex,
                          Warning)
            depth = 0
    rcParams['mathtext.fontset'] = orig_fontset
    sys.stdout.write("#")
    sys.stdout.flush()
    return depth

# LaTeX to HTML translation stuff:
def latex2html(node, source):
    inline = isinstance(node.parent, nodes.TextElement)
    latex = node['latex']
    name = 'math-%s' % md5(latex.encode()).hexdigest()[-10:]

    destdir = os.path.join(setup.app.builder.outdir, '_images', 'mathmpl')
    if not os.path.exists(destdir):
        os.makedirs(destdir)
    dest = os.path.join(destdir, '%s.png' % name)
    path = '/'.join((setup.app.builder.imgpath, 'mathmpl'))

    depth = latex2png(latex, dest, node['fontset'])

    if inline:
        cls = ''
    else:
        cls = 'class="center" '
    if inline and depth != 0:
        style = 'style="position: relative; bottom: -%dpx"' % (depth + 1)
    else:
        style = ''

    return '<img src="%s/%s.png" %s%s/>' % (path, name, cls, style)

def setup(app):
    setup.app = app

    # Add visit/depart methods to HTML-Translator:
    def visit_latex_math_html(self, node):
        source = self.document.attributes['source']
        self.body.append(latex2html(node, source))

    def depart_latex_math_html(self, node):
        pass

    # Add visit/depart methods to LaTeX-Translator:
    def visit_latex_math_latex(self, node):
        inline = isinstance(node.parent, nodes.TextElement)
        if inline:
            self.body.append('$%s$' % node['latex'])
        else:
            self.body.extend(['\\begin{equation}',
                              node['latex'],
                              '\\end{equation}'])

    def depart_latex_math_latex(self, node):
        pass

    app.add_node(latex_math,
                 html=(visit_latex_math_html, depart_latex_math_html),
                 latex=(visit_latex_math_latex, depart_latex_math_latex))
    app.add_role('math', math_role)
    app.add_directive('math', math_directive,
                      True, (0, 0, 0), **options_spec)

    metadata = {'parallel_read_safe': True, 'parallel_write_safe': True}
    return metadata


#!/usr/bin/python
#-*- coding:Utf-8 -*-

import matplotlib.pyplot as plt
import numpy as np
import networkx as nx
from pylayers.gis.layout import *
from pylayers.antprop.signature import *

# load the layout graphs
def showr2(L,r2d,tx,rx,k,l):
    col = ['r','b','g','c','m','k','y']
    r = r2d[str(k)]
    pts = r['pt']
    sig = r['sig']
    fig,ax = showsig(L,sig[:,:,l],tx,rx)
    sh = np.shape(pts)
    x = np.hstack((tx[0],pts[0,:,l],rx[0]))
    y = np.hstack((tx[1],pts[1,:,l],rx[1]))
    plt.plot(x,y,col[k])
    plt.title(sig[:,:,l])
    return fig,ax

def showr2d(L,r2d,tx,rx):
    """
    r2d['pt'] : nd,ni,nr
    """
    L.display['thin']=True
    col = ['r','b','g','c','m','k','y']
    fig,ax = L.showGs()
    for k in r2d:
        r = r2d[k]
        pts = r['pt']
        sh = np.shape(pts)
        for r in range(sh[2]):
            x = np.hstack((tx[0],pts[0,:,r],rx[0]))
            y = np.hstack((tx[1],pts[1,:,r],rx[1]))
            plt.plot(x,y,col[eval(k)])
    return fig,ax

def showsig(L,s,tx,rx):
    L.display['thin']=True
    fig,ax = L.showGs()
    L.display['thin']=False
    L.display['edlabel']=True
    fig,ax = L.showGs(fig=fig,ax=ax,edlist=s[0,:],width=4)
    plt.plot(tx[0],tx[1],'x')
    plt.plot(rx[0],rx[1],'+')
    plt.title(str(s[0,:])+str(s[1,:]))
    L.display['edlabel']=False
    return fig,ax

strucname = 'TA-Office'
#strucname = 'defstr'
L = Layout(strucname+'.ini')
L.boundary()
print L.ax
try:
    L.dumpr()
except:
    L.build()
    L.dumpw()
#tx = np.array([8., 8., 1.])
#rx = np.array([30., 11., 2.])
#tx = np.array([1., 0., 1.])
#rx = np.array([8., -1.5, 2.])

#L = Layout('TA-Office.str')
#L.build()
tx = np.array([20, 8, 1])
rx = np.array([35, 6, 2])

S = Signatures(L, tx, rx)

print "Calcul signatures"
#s1 = S.get_sigslist(tx, rx)
s1 = S.run(tx,rx,2)
print "Fin calcul signatures"

#print "signatures --> rayons "
#r2d = S.sigs2rays(s1)
r2d = S.rays(s1)
##print "fin signatures --> rayons "
##
#r22 = r2d['2']
#pt2 = r22['pt']
#sig2 = r22['sig']
#pt2 = np.swapaxes(pt2,0,2)
#pt2 = np.swapaxes(pt2,1,2)
#tx2 = np.kron(np.ones(2),tx).reshape(2,3,1)
#rx2 = np.kron(np.ones(2),rx).reshape(2,3,1)
#tx2[:,2,:]=0
#rx2[:,2,:]=0
#pt  = np.concatenate((tx2,pt2,rx2),axis=2)
#vsi = pt[:, :, 1:] - pt[:,:,:-1]
#si  = np.sqrt(np.sum(vsi*vsi, axis=1))
#alpha = np.cumsum(si,axis=1)
#c  = alpha[:,-1].reshape(2,1)
#alpha = alpha/c
#pt[:,2,1:]= tx[2]+alpha*(rx[2]-tx[2])
#
#
showr2d(L,r2d,tx,rx)
print "rayons 2D --> rayons3D "
#rays3d = S.ray2D3D(r2d)
#print "fin rayons 2D --> rayons3D "
##
#S.show3(rays=rays3d,strucname=strucname)
##
##
##
#s    = np.array([[5,1,8],[1,1,2]])
#sig  = Signature(s)
#rsig = sig.sig2ray(L,tx[0:2],rx[0:2])
#sig.ev(L)
#M = sig.image(tx[0:2])
#Y = sig.backtrace(tx[0:2],rx[0:2],M)
#plt.plot(M[0,:],M[1,:],'ob')
#plt.plot(Y[0,:],Y[1,:],'xk')
#fig,ax = showr2(L,r2d,tx[0:2],rx[0:2],3,4)
#plt.show()
#room8 = L.Gt.node[8]
#polyg8 = room8['polyg']
#vnodes8 = room8['vnodes']
#udeg1 = []
#udeg2 = []
#for ik, inode in enumerate(vnodes8):
#    deg = L.Gs.degree(inode)
#    if vnodes8[0] < 0:
#        index = ik / 2
#    else:
#        index = (ik - 1) / 2
#    if inode < 0:
#        if deg == 2:
#            udeg2.append(index)
#        if deg == 1:
#            udeg1.append(index)    # warning not used
#Gv = polyg8.buildGv(show=True,udeg2=udeg2)
#L.showGs()
#nx.draw_networkx_edges(L.dGv[8],L.Gs.pos,nx.edges(L.dGv[8],nbunch=[47]))


# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests feeding functions using arrays and `DataFrames`."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

import numpy as np

from tensorflow.python.estimator.inputs.queues import feeding_functions as ff
from tensorflow.python.platform import test

try:
  # pylint: disable=g-import-not-at-top
  import pandas as pd
  HAS_PANDAS = True
except IOError:
  # Pandas writes a temporary file during import. If it fails, don't use pandas.
  HAS_PANDAS = False
except ImportError:
  HAS_PANDAS = False

def vals_to_list(a):
  return {
      key: val.tolist() if isinstance(val, np.ndarray) else val
      for key, val in a.items()
  }

class _FeedingFunctionsTestCase(test.TestCase):
  """Tests for feeding functions."""

  def testArrayFeedFnBatchOne(self):
    array = np.arange(32).reshape([16, 2])
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, 1)

    # cycle around a couple times
    for x in range(0, 100):
      i = x % 16
      expected = {
          "index_placeholder": [i],
          "value_placeholder": [[2 * i, 2 * i + 1]]
      }
      actual = aff()
      self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchFive(self):
    array = np.arange(32).reshape([16, 2])
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, 5)

    # cycle around a couple times
    for _ in range(0, 101, 2):
      aff()

    expected = {
        "index_placeholder": [15, 0, 1, 2, 3],
        "value_placeholder": [[30, 31], [0, 1], [2, 3], [4, 5], [6, 7]]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchTwoWithOneEpoch(self):
    array = np.arange(5) + 10
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, batch_size=2, num_epochs=1)

    expected = {
        "index_placeholder": [0, 1],
        "value_placeholder": [10, 11]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [2, 3],
        "value_placeholder": [12, 13]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [4],
        "value_placeholder": [14]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchOneHundred(self):
    array = np.arange(32).reshape([16, 2])
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, 100)

    expected = {
        "index_placeholder":
            list(range(0, 16)) * 6 + list(range(0, 4)),
        "value_placeholder":
            np.arange(32).reshape([16, 2]).tolist() * 6 +
            [[0, 1], [2, 3], [4, 5], [6, 7]]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchOneHundredWithSmallerArrayAndMultipleEpochs(self):
    array = np.arange(2) + 10
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, batch_size=100, num_epochs=2)

    expected = {
        "index_placeholder": [0, 1, 0, 1],
        "value_placeholder": [10, 11, 10, 11],
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchOne(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 64)
    array2 = np.arange(64, 96)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 128))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, 1)

    # cycle around a couple times
    for x in range(0, 100):
      i = x % 32
      expected = {
          "index_placeholder": [i + 96],
          "a_placeholder": [32 + i],
          "b_placeholder": [64 + i]
      }
      actual = aff()
      self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchFive(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 64)
    array2 = np.arange(64, 96)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 128))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, 5)

    # cycle around a couple times
    for _ in range(0, 101, 2):
      aff()

    expected = {
        "index_placeholder": [127, 96, 97, 98, 99],
        "a_placeholder": [63, 32, 33, 34, 35],
        "b_placeholder": [95, 64, 65, 66, 67]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchTwoWithOneEpoch(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 37)
    array2 = np.arange(64, 69)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 101))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, batch_size=2, num_epochs=1)

    expected = {
        "index_placeholder": [96, 97],
        "a_placeholder": [32, 33],
        "b_placeholder": [64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [98, 99],
        "a_placeholder": [34, 35],
        "b_placeholder": [66, 67]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [100],
        "a_placeholder": [36],
        "b_placeholder": [68]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchOneHundred(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 64)
    array2 = np.arange(64, 96)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 128))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, 100)

    expected = {
        "index_placeholder": list(range(96, 128)) * 3 + list(range(96, 100)),
        "a_placeholder": list(range(32, 64)) * 3 + list(range(32, 36)),
        "b_placeholder": list(range(64, 96)) * 3 + list(range(64, 68))
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchOneHundredWithSmallDataArrayAndMultipleEpochs(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 34)
    array2 = np.arange(64, 66)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 98))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, batch_size=100, num_epochs=2)

    expected = {
        "index_placeholder": [96, 97, 96, 97],
        "a_placeholder": [32, 33, 32, 33],
        "b_placeholder": [64, 65, 64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testOrderedDictNumpyFeedFnBatchTwoWithOneEpoch(self):
    a = np.arange(32, 37)
    b = np.arange(64, 69)
    x = {"a": a, "b": b}
    ordered_dict_x = collections.OrderedDict(
        sorted(x.items(), key=lambda t: t[0]))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._OrderedDictNumpyFeedFn(
        placeholders, ordered_dict_x, batch_size=2, num_epochs=1)

    expected = {
        "index_placeholder": [0, 1],
        "a_placeholder": [32, 33],
        "b_placeholder": [64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [2, 3],
        "a_placeholder": [34, 35],
        "b_placeholder": [66, 67]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [4],
        "a_placeholder": [36],
        "b_placeholder": [68]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testOrderedDictNumpyFeedFnLargeBatchWithSmallArrayAndMultipleEpochs(self):
    a = np.arange(32, 34)
    b = np.arange(64, 66)
    x = {"a": a, "b": b}
    ordered_dict_x = collections.OrderedDict(
        sorted(x.items(), key=lambda t: t[0]))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._OrderedDictNumpyFeedFn(
        placeholders, ordered_dict_x, batch_size=100, num_epochs=2)

    expected = {
        "index_placeholder": [0, 1, 0, 1],
        "a_placeholder": [32, 33, 32, 33],
        "b_placeholder": [64, 65, 64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testFillArraySmall(self):
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[64, 36], dtype=np.int32)
    ff._fill_array(actual, a)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testFillArrayLarge(self):
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    ff._fill_array(actual, a)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testFillArraySmallWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[64, 36], dtype=np.int32)
    ff._fill_array(actual, a, fill_value)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testFillArrayLargeWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    ff._fill_array(actual, a, fill_value)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededSmall(self):
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededLarge(self):
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededSmallWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededLargeWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededSmallWithSpecifiedNonNumericValue(self):
    fill_value = False
    a = (np.ones(shape=[32, 32], dtype=np.bool).tolist() +
         np.ones(shape=[32, 36], dtype=np.bool).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[64, 36], dtype=np.bool)
    expected[:32, 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededLargeWithSpecifiedNonNumericValue(self):
    fill_value = False
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.bool).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.bool).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.bool)
    expected[:8, ..., 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

if __name__ == "__main__":
  test.main()


from matplotlib.collections import LineCollection
import matplotlib.pyplot as plt
import numpy as np

from bokeh import mpl
from bokeh.plotting import output_file, show

def make_segments(x, y):
    '''
    Create list of line segments from x and y coordinates.
    '''

    points = np.array([x, y]).T.reshape(-1, 1, 2)
    segments = np.concatenate([points[:-1], points[1:]], axis=1)

    return segments

def colorline(x, y, colors=None, linewidth=3, alpha=1.0):
    '''
    Plot a line with segments.
    Optionally, specify segments colors and segments widths.
    '''

    # Make a list of colors cycling through the rgbcmyk series.
    # You have several ways to input the colors:
    # colors = ['r','g','b','c','y','m','k']
    # colors = ['red','green','blue','cyan','yellow','magenta','black']
    # colors = ['#ff0000', '#008000', '#0000ff', '#00bfbf', '#bfbf00', '#bf00bf', '#000000']
    # colors = [(1.0, 0.0, 0.0, 1.0), (0.0, 0.5, 0.0, 1.0), (0.0, 0.0, 1.0, 1.0), (0.0, 0.75, 0.75, 1.0),
    #           (0.75, 0.75, 0, 1.0), (0.75, 0, 0.75, 1.0), (0.0, 0.0, 0.0, 1.0)]

    colors = ['r', 'g', 'b', 'c', 'y', 'm', 'k']
    widths = [5, 10, 20, 40, 20, 10, 5]

    segments = make_segments(x, y)
    lc = LineCollection(segments, colors=colors, linewidth=widths, alpha=alpha)

    ax = plt.gca()
    ax.add_collection(lc)

    return lc

# Colored sine wave

x = np.linspace(0, 4 * np.pi, 100)
y = np.sin(x)

colorline(x, y)

plt.title("MPL support for ListCollection in Bokeh")
plt.xlim(x.min(), x.max())
plt.ylim(-1.0, 1.0)

output_file("listcollection.html")

show(mpl.to_bokeh())


"""
Tests for DBSCAN clustering algorithm
"""

import pickle

import numpy as np
from numpy.testing import assert_equal
from scipy.spatial import distance

from sklearn.cluster.dbscan_ import DBSCAN, dbscan
from .common import generate_clustered_data

n_clusters = 3
X = generate_clustered_data(n_clusters=n_clusters)

def test_dbscan_similarity():
    """Tests the DBSCAN algorithm with a similarity array."""
    # Parameters chosen specifically for this task.
    eps = 0.15
    min_samples = 10
    # Compute similarities
    D = distance.squareform(distance.pdist(X))
    D /= np.max(D)
    # Compute DBSCAN
    core_samples, labels = dbscan(D, metric="precomputed",
                                 eps=eps, min_samples=min_samples)
    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - (1 if -1 in labels else 0)

    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric="precomputed", eps=eps, min_samples=min_samples)
    labels = db.fit(D).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

def test_dbscan_feature():
    """Tests the DBSCAN algorithm with a feature vector array."""
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    metric = 'euclidean'
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(X, metric=metric,
                                 eps=eps, min_samples=min_samples)

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples)
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

def test_dbscan_callable():
    """Tests the DBSCAN algorithm with a callable metric."""
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    # metric is the function reference, not the string key.
    metric = distance.euclidean
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(X, metric=metric,
                                 eps=eps, min_samples=min_samples)

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples)
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

def test_pickle():
    obj = DBSCAN()
    s = pickle.dumps(obj)
    assert_equal(type(pickle.loads(s)), obj.__class__)


#!/usr/bin/env python

import argparse
import os
import re
import shutil

import pandas
import pysam
from Bio import SeqIO

def get_sample_name(file_path):
    base_file_name = os.path.basename(file_path)
    if base_file_name.find(".") > 0:
        # Eliminate the extension.
        return os.path.splitext(base_file_name)[0]
    return base_file_name

def get_coverage_df(bam_file):
    # Create a coverage dictionary.
    coverage_dict = {}
    coverage_list = pysam.depth(bam_file, split_lines=True)
    for line in coverage_list:
        chrom, position, depth = line.split('\t')
        coverage_dict["%s-%s" % (chrom, position)] = depth
    # Convert it to a data frame.
    coverage_df = pandas.DataFrame.from_dict(coverage_dict, orient='index', columns=["depth"])
    return coverage_df

def get_zero_df(reference):
    # Create a zero coverage dictionary.
    zero_dict = {}
    for record in SeqIO.parse(reference, "fasta"):
        chrom = record.id
        total_len = len(record.seq)
        for pos in list(range(1, total_len + 1)):
            zero_dict["%s-%s" % (str(chrom), str(pos))] = 0
    # Convert it to a data frame with depth_x
    # and depth_y columns - index is NaN.
    zero_df = pandas.DataFrame.from_dict(zero_dict, orient='index', columns=["depth"])
    return zero_df

def output_zc_vcf_file(base_file_name, vcf_file, zero_df, total_zero_coverage, output_vcf):
    column_names = ["CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT", "Sample"]
    vcf_df = pandas.read_csv(vcf_file, sep='\t', header=None, names=column_names, comment='#')
    good_snp_count = len(vcf_df[(vcf_df['ALT'].str.len() == 1) & (vcf_df['REF'].str.len() == 1) & (vcf_df['QUAL'] > 150)])
    if total_zero_coverage > 0:
        header_file = "%s_header.csv" % base_file_name
        with open(header_file, 'w') as outfile:
            with open(vcf_file) as infile:
                for line in infile:
                    if re.search('^#', line):
                        outfile.write("%s" % line)
        vcf_df_snp = vcf_df[vcf_df['REF'].str.len() == 1]
        vcf_df_snp = vcf_df_snp[vcf_df_snp['ALT'].str.len() == 1]
        vcf_df_snp['ABS_VALUE'] = vcf_df_snp['CHROM'].map(str) + "-" + vcf_df_snp['POS'].map(str)
        vcf_df_snp = vcf_df_snp.set_index('ABS_VALUE')
        cat_df = pandas.concat([vcf_df_snp, zero_df], axis=1, sort=False)
        cat_df = cat_df.drop(columns=['CHROM', 'POS', 'depth'])
        cat_df[['ID', 'ALT', 'QUAL', 'FILTER', 'INFO']] = cat_df[['ID', 'ALT', 'QUAL', 'FILTER', 'INFO']].fillna('.')
        cat_df['REF'] = cat_df['REF'].fillna('N')
        cat_df['FORMAT'] = cat_df['FORMAT'].fillna('GT')
        cat_df['Sample'] = cat_df['Sample'].fillna('./.')
        cat_df['temp'] = cat_df.index.str.rsplit('-', n=1)
        cat_df[['CHROM', 'POS']] = pandas.DataFrame(cat_df.temp.values.tolist(), index=cat_df.index)
        cat_df = cat_df[['CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT', 'Sample']]
        cat_df['POS'] = cat_df['POS'].astype(int)
        cat_df = cat_df.sort_values(['CHROM', 'POS'])
        body_file = "%s_body.csv" % base_file_name
        cat_df.to_csv(body_file, sep='\t', header=False, index=False)
        with open(output_vcf, "w") as outfile:
            for cf in [header_file, body_file]:
                with open(cf, "r") as infile:
                    for line in infile:
                        outfile.write("%s" % line)
    else:
        shutil.move(vcf_file, output_vcf)
    return good_snp_count

def output_metrics_file(base_file_name, average_coverage, genome_coverage, good_snp_count, output_metrics):
    bam_metrics = [base_file_name, "", "%4f" % average_coverage, genome_coverage]
    vcf_metrics = [base_file_name, str(good_snp_count), "", ""]
    metrics_columns = ["File", "Number of Good SNPs", "Average Coverage", "Genome Coverage"]
    with open(output_metrics, "w") as fh:
        fh.write("# %s\n" % "\t".join(metrics_columns))
        fh.write("%s\n" % "\t".join(bam_metrics))
        fh.write("%s\n" % "\t".join(vcf_metrics))

def output_files(vcf_file, total_zero_coverage, zero_df, output_vcf, average_coverage, genome_coverage, output_metrics):
    base_file_name = get_sample_name(vcf_file)
    good_snp_count = output_zc_vcf_file(base_file_name, vcf_file, zero_df, total_zero_coverage, output_vcf)
    output_metrics_file(base_file_name, average_coverage, genome_coverage, good_snp_count, output_metrics)

def get_coverage_and_snp_count(bam_file, vcf_file, reference, output_metrics, output_vcf):
    coverage_df = get_coverage_df(bam_file)
    zero_df = get_zero_df(reference)
    coverage_df = zero_df.merge(coverage_df, left_index=True, right_index=True, how='outer')
    # depth_x "0" column no longer needed.
    coverage_df = coverage_df.drop(columns=['depth_x'])
    coverage_df = coverage_df.rename(columns={'depth_y': 'depth'})
    # Covert the NaN to 0 coverage and get some metrics.
    coverage_df = coverage_df.fillna(0)
    coverage_df['depth'] = coverage_df['depth'].apply(int)
    total_length = len(coverage_df)
    average_coverage = coverage_df['depth'].mean()
    zero_df = coverage_df[coverage_df['depth'] == 0]
    total_zero_coverage = len(zero_df)
    total_coverage = total_length - total_zero_coverage
    genome_coverage = "{:.2%}".format(total_coverage / total_length)
    # Output a zero-coverage vcf fil and the metrics file.
    output_files(vcf_file, total_zero_coverage, zero_df, output_vcf, average_coverage, genome_coverage, output_metrics)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--bam_input', action='store', dest='bam_input', help='bam input file')
    parser.add_argument('--output_metrics', action='store', dest='output_metrics', required=False, default=None, help='Output metrics text file')
    parser.add_argument('--output_vcf', action='store', dest='output_vcf', required=False, default=None, help='Output VCF file')
    parser.add_argument('--reference', action='store', dest='reference', help='Reference dataset')
    parser.add_argument('--vcf_input', action='store', dest='vcf_input', help='vcf input file')

    args = parser.parse_args()

    get_coverage_and_snp_count(args.bam_input, args.vcf_input, args.reference, args.output_metrics, args.output_vcf)


#!/bin/python3

import numpy as np
import cv2
import base64
import pdb
from tkinter import *
from matplotlib import pyplot as plt

class FeatureMatcher:

  __PORC_DISTANCE = 0.7

  def __init__(self,feature_extractor='SURF',upright=True,min_match_count=10,threshold=400):
    self.MIN_MATCH_COUNT = min_match_count
    self.__create_feature_extractor(feature_extractor,upright,threshold)

    FLANN_INDEX_KDTREE = 0
    index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
    search_params = dict(checks = 200)
    self.flann = cv2.FlannBasedMatcher(index_params, search_params)

  def __create_feature_extractor(self,feature_extractor,upright,threshold):
    if feature_extractor == 'SURF':
      self.feature_finder = cv2.xfeatures2d.SURF_create(threshold,extended=True)
      self.feature_finder.setUpright(upright)
    elif feature_extractor == 'SIFT':
      self.feature_finder = cv2.xfeatures2d.SIFT_create(edgeThreshold=20,sigma=1.1)
    elif feature_extractor == 'ORB':
      self.feature_finder = cv2.ORB_create()
    else:
      raise 'Feature extractor no encontrado'

  def compare(self,img1,img2):
    self.features_img1 = self.find_features(img1)
    self.features_img2 = self.find_features(img2)
    pdb.set_trace()
    return self.flann.knnMatch(self.features_img1[1],self.features_img2[1],k=2)

  def compare_base64(self,image1_base64,image2_base64):
    img1 = self.base64_to_img(image1_base64)
    img2 = self.base64_to_img(image2_base64)

    return self.compare(img1,img2)

  def are_similar(self,img1,img2):
    self.good_matches = []

    for m,n in self.compare(img1,img2):
      if m.distance < self.__PORC_DISTANCE*n.distance:
        self.good_matches.append(m)

    return (len(self.good_matches) > self.MIN_MATCH_COUNT)

  def find_features(self,img):
    return self.feature_finder.detectAndCompute(img,None)

  def bytes_to_img(self,image_bytes):
    nparr = np.fromstring(image_bytes, np.uint8)
    return cv2.imdecode(nparr, 0)

  def base64_to_img(self,image_base64):
    return self.bytes_to_img(base64.b64decode(image_base64))

  def compare_and_draw_base64(self,img1,img2):
    self.compare_and_draw(self.base64_to_img(img1),self.base64_to_img(img2))

  def compare_and_draw(self,img1,img2):
    # if self.are_similar(img1,img2):
    #   src_pts = np.float32([ self.features_img1[0][m.queryIdx].pt for m in self.good_matches ]).reshape(-1,1,2)
    #   dst_pts = np.float32([ self.features_img2[0][m.trainIdx].pt for m in self.good_matches ]).reshape(-1,1,2)

    #   M, mask = cv2.findHomography(src_pts,dst_pts,cv2.RANSAC,5.0)
    #   matchesMask = mask.ravel().tolist()

    #   h,w = img1.shape
    #   pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
    #   dst = cv2.perspectiveTransform(pts,M)

    #   img2 = cv2.polylines(img2,[np.int32(dst)],True,255,3,cv2.LINE_AA)
    # else:
    #   print("Not enough matches are found - %d/%d" % (len(self.good_matches),self.MIN_MATCH_COUNT))
    #   matchesMask = None

    # draw_params = dict(matchColor = (0,255,0),
    #                    singlePointColor = (255,0,0),
    #                    matchesMask = matchesMask,
    #                    flags = 2)

    # img3 = cv2.drawMatchesKnn(img1,self.features_img1[0],img2,self.features_img2[0],self.good_matches,None,**draw_params)

    # plt.imshow(img3,'gray'),plt.show()
    hash1 = self.find_features(img1)
    hash2 = self.find_features(img2)
    matches = self.flann.knnMatch(hash1[1],hash2[1],k=2)

    good = []
    for m,n in matches:
      if m.distance < 0.95*n.distance:
          good.append(m)
    print(len(good))

    if len(good)>self.MIN_MATCH_COUNT:
      src_pts = np.float32([ hash1[0][m.queryIdx].pt for m in good ]).reshape(-1,1,2)
      dst_pts = np.float32([ hash2[0][m.trainIdx].pt for m in good ]).reshape(-1,1,2)
      M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
      matchesMask = mask.ravel().tolist()
      h,w = img1.shape
      pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
      dst = cv2.perspectiveTransform(pts,M)
      img2 = cv2.polylines(img2,[np.int32(dst)],True,255,3, cv2.LINE_AA)
    else:
      print( "Not enough matches are found - {}/{}".format(len(good), self.MIN_MATCH_COUNT) )
      matchesMask = None

    draw_params = dict(matchColor = (0,255,0), # draw matches in green color
                       singlePointColor = (255,0,0),
                       matchesMask = matchesMask, # draw only inliers
                       flags = 2)

    img3 = cv2.drawMatches(img1,hash1[0],img2,hash2[0],good,None,**draw_params)
    plt.imshow(img3, 'gray'),plt.show()


import pytest

import matplotlib.pyplot as plt

from aneris.control.factory import InterfaceFactory
from dtocean_core.core import (AutoFileInput,
                               AutoFileOutput,
                               AutoPlot,
                               Core)
from dtocean_core.data import CoreMetaData
from dtocean_core.data.definitions import SimplePie

def test_SimplePie_available():

    new_core = Core()
    all_objs = new_core.control._store._structures

    assert "SimplePie" in all_objs.keys()

def test_SimplePie():

    meta = CoreMetaData({"identifier": "test",
                         "structure": "test",
                         "title": "test",
                         "types": ["float"]})

    test = SimplePie()

    raw = {"a": 0, "b": 1}
    a = test.get_data(raw, meta)
    b = test.get_value(a)

    assert b["a"] == 0
    assert b["b"] == 1

def test_get_None():

    test = SimplePie()
    result = test.get_value(None)

    assert result is None

@pytest.mark.parametrize("fext", [".csv", ".xls", ".xlsx"])
def test_SimplePie_auto_file(tmpdir, fext):

    test_path = tmpdir.mkdir("sub").join("test{}".format(fext))
    test_path_str = str(test_path)

    raw = {"a": 0, "b": 1}

    meta = CoreMetaData({"identifier": "test",
                         "structure": "test",
                         "title": "test",
                         "types": ["float"]})

    test = SimplePie()

    fout_factory = InterfaceFactory(AutoFileOutput)
    FOutCls = fout_factory(meta, test)

    fout = FOutCls()
    fout._path = test_path_str
    fout.data.result = test.get_data(raw, meta)

    fout.connect()

    assert len(tmpdir.listdir()) == 1

    fin_factory = InterfaceFactory(AutoFileInput)
    FInCls = fin_factory(meta, test)

    fin = FInCls()
    fin._path = test_path_str

    fin.connect()
    result = test.get_data(fin.data.result, meta)

    assert result["a"] == 0
    assert result["b"] == 1

def test_SimplePie_auto_plot():

    raw = {"a": 0, "b": 1}

    meta = CoreMetaData({"identifier": "test",
                         "structure": "test",
                         "title": "test",
                         "types": ["float"]})

    test = SimplePie()

    fout_factory = InterfaceFactory(AutoPlot)
    PlotCls = fout_factory(meta, test)

    plot = PlotCls()
    plot.data.result = test.get_data(raw, meta)
    plot.meta.result = meta

    plot.connect()

    assert len(plt.get_fignums()) == 1
    plt.close("all")


# Copyright (c) 2012-2014, GPy authors (see AUTHORS.txt).
# Licensed under the BSD 3-clause license (see LICENSE.txt)

import numpy as np
import sys
from .. import kern
from .model import Model
from .parameterization import ObsAr
from .mapping import Mapping
from .. import likelihoods
from ..inference.latent_function_inference import exact_gaussian_inference, expectation_propagation
from .parameterization.variational import VariationalPosterior

import logging
import warnings
from GPy.util.normalizer import MeanNorm
logger = logging.getLogger("GP")

class GP(Model):
    """
    General purpose Gaussian process model

    :param X: input observations
    :param Y: output observations
    :param kernel: a GPy kernel, defaults to rbf+white
    :param likelihood: a GPy likelihood
    :param inference_method: The :class:`~GPy.inference.latent_function_inference.LatentFunctionInference` inference method to use for this GP
    :rtype: model object
    :param Norm normalizer:
        normalize the outputs Y.
        Prediction will be un-normalized using this normalizer.
        If normalizer is None, we will normalize using MeanNorm.
        If normalizer is False, no normalization will be done.

    .. Note:: Multiple independent outputs are allowed using columns of Y

    """
    def __init__(self, X, Y, kernel, likelihood, mean_function=None, inference_method=None, name='gp', Y_metadata=None, normalizer=False):
        super(GP, self).__init__(name)

        assert X.ndim == 2
        if isinstance(X, (ObsAr, VariationalPosterior)):
            self.X = X.copy()
        else: self.X = ObsAr(X)

        self.num_data, self.input_dim = self.X.shape

        assert Y.ndim == 2
        logger.info("initializing Y")

        if normalizer is True:
            self.normalizer = MeanNorm()
        elif normalizer is False:
            self.normalizer = None
        else:
            self.normalizer = normalizer

        if self.normalizer is not None:
            self.normalizer.scale_by(Y)
            self.Y_normalized = ObsAr(self.normalizer.normalize(Y))
            self.Y = Y
        elif isinstance(Y, np.ndarray):
            self.Y = ObsAr(Y)
            self.Y_normalized = self.Y
        else:
            self.Y = Y

        if Y.shape[0] != self.num_data:
            #There can be cases where we want inputs than outputs, for example if we have multiple latent
            #function values
            warnings.warn("There are more rows in your input data X, \
                         than in your output data Y, be VERY sure this is what you want")
        _, self.output_dim = self.Y.shape

        assert ((Y_metadata is None) or isinstance(Y_metadata, dict))
        self.Y_metadata = Y_metadata

        assert isinstance(kernel, kern.Kern)
        #assert self.input_dim == kernel.input_dim
        self.kern = kernel

        assert isinstance(likelihood, likelihoods.Likelihood)
        self.likelihood = likelihood

        #handle the mean function
        self.mean_function = mean_function
        if mean_function is not None:
            assert isinstance(self.mean_function, Mapping)
            assert mean_function.input_dim == self.input_dim
            assert mean_function.output_dim == self.output_dim
            self.link_parameter(mean_function)

        #find a sensible inference method
        logger.info("initializing inference method")
        if inference_method is None:
            if isinstance(likelihood, likelihoods.Gaussian) or isinstance(likelihood, likelihoods.MixedNoise):
                inference_method = exact_gaussian_inference.ExactGaussianInference()
            else:
                inference_method = expectation_propagation.EP()
                print("defaulting to ", inference_method, "for latent function inference")
        self.inference_method = inference_method

        logger.info("adding kernel and likelihood as parameters")
        self.link_parameter(self.kern)
        self.link_parameter(self.likelihood)
        self.posterior = None

        # The predictive variable to be used to predict using the posterior object's
        # woodbury_vector and woodbury_inv is defined as predictive_variable
        # as long as the posterior has the right woodbury entries.
        # It is the input variable used for the covariance between
        # X_star and the posterior of the GP.
        # This is usually just a link to self.X (full GP) or self.Z (sparse GP).
        # Make sure to name this variable and the predict functions will "just work"
        # In maths the predictive variable is:
        #         K_{xx} - K_{xp}W_{pp}^{-1}K_{px}
        #         W_{pp} := \texttt{Woodbury inv}
        #         p := _predictive_variable

    @property
    def _predictive_variable(self):
        return self.X

    def set_XY(self, X=None, Y=None):
        """
        Set the input / output data of the model
        This is useful if we wish to change our existing data but maintain the same model

        :param X: input observations
        :type X: np.ndarray
        :param Y: output observations
        :type Y: np.ndarray
        """
        self.update_model(False)
        if Y is not None:
            if self.normalizer is not None:
                self.normalizer.scale_by(Y)
                self.Y_normalized = ObsAr(self.normalizer.normalize(Y))
                self.Y = Y
            else:
                self.Y = ObsAr(Y)
                self.Y_normalized = self.Y
        if X is not None:
            if self.X in self.parameters:
                # LVM models
                if isinstance(self.X, VariationalPosterior):
                    assert isinstance(X, type(self.X)), "The given X must have the same type as the X in the model!"
                    self.unlink_parameter(self.X)
                    self.X = X
                    self.link_parameter(self.X)
                else:
                    self.unlink_parameter(self.X)
                    from ..core import Param
                    self.X = Param('latent mean',X)
                    self.link_parameter(self.X)
            else:
                self.X = ObsAr(X)
        self.update_model(True)

    def set_X(self,X):
        """
        Set the input data of the model

        :param X: input observations
        :type X: np.ndarray
        """
        self.set_XY(X=X)

    def set_Y(self,Y):
        """
        Set the output data of the model

        :param X: output observations
        :type X: np.ndarray
        """
        self.set_XY(Y=Y)

    def parameters_changed(self):
        """
        Method that is called upon any changes to :class:`~GPy.core.parameterization.param.Param` variables within the model.
        In particular in the GP class this method reperforms inference, recalculating the posterior and log marginal likelihood and gradients of the model

        .. warning::
            This method is not designed to be called manually, the framework is set up to automatically call this method upon changes to parameters, if you call
            this method yourself, there may be unexpected consequences.
        """
        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.likelihood, self.Y_normalized, self.mean_function, self.Y_metadata)
        self.likelihood.update_gradients(self.grad_dict['dL_dthetaL'])
        self.kern.update_gradients_full(self.grad_dict['dL_dK'], self.X)
        if self.mean_function is not None:
            self.mean_function.update_gradients(self.grad_dict['dL_dm'], self.X)

    def log_likelihood(self):
        """
        The log marginal likelihood of the model, :math:`p(\mathbf{y})`, this is the objective function of the model being optimised
        """
        return self._log_marginal_likelihood

    def _raw_predict(self, Xnew, full_cov=False, kern=None):
        """
        For making predictions, does not account for normalization or likelihood

        full_cov is a boolean which defines whether the full covariance matrix
        of the prediction is computed. If full_cov is False (default), only the
        diagonal of the covariance is returned.

        .. math::
            p(f*|X*, X, Y) = \int^{\inf}_{\inf} p(f*|f,X*)p(f|X,Y) df
                        = N(f*| K_{x*x}(K_{xx} + \Sigma)^{-1}Y, K_{x*x*} - K_{xx*}(K_{xx} + \Sigma)^{-1}K_{xx*}
            \Sigma := \texttt{Likelihood.variance / Approximate likelihood covariance}
        """
        if kern is None:
            kern = self.kern

        Kx = kern.K(self._predictive_variable, Xnew)
        mu = np.dot(Kx.T, self.posterior.woodbury_vector)
        if len(mu.shape)==1:
            mu = mu.reshape(-1,1)
        if full_cov:
            Kxx = kern.K(Xnew)
            if self.posterior.woodbury_inv.ndim == 2:
                var = Kxx - np.dot(Kx.T, np.dot(self.posterior.woodbury_inv, Kx))
            elif self.posterior.woodbury_inv.ndim == 3: # Missing data
                var = np.empty((Kxx.shape[0],Kxx.shape[1],self.posterior.woodbury_inv.shape[2]))
                from ..util.linalg import mdot
                for i in range(var.shape[2]):
                    var[:, :, i] = (Kxx - mdot(Kx.T, self.posterior.woodbury_inv[:, :, i], Kx))
            var = var
        else:
            Kxx = kern.Kdiag(Xnew)
            if self.posterior.woodbury_inv.ndim == 2:
                var = (Kxx - np.sum(np.dot(self.posterior.woodbury_inv.T, Kx) * Kx, 0))[:,None]
            elif self.posterior.woodbury_inv.ndim == 3: # Missing data
                var = np.empty((Kxx.shape[0],self.posterior.woodbury_inv.shape[2]))
                for i in range(var.shape[1]):
                    var[:, i] = (Kxx - (np.sum(np.dot(self.posterior.woodbury_inv[:, :, i].T, Kx) * Kx, 0)))
            var = var
        #add in the mean function
        if self.mean_function is not None:
            mu += self.mean_function.f(Xnew)

        return mu, var

    def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None):
        """
        Predict the function(s) at the new point(s) Xnew.

        :param Xnew: The points at which to make a prediction
        :type Xnew: np.ndarray (Nnew x self.input_dim)
        :param full_cov: whether to return the full covariance matrix, or just
                         the diagonal
        :type full_cov: bool
        :param Y_metadata: metadata about the predicting point to pass to the likelihood
        :param kern: The kernel to use for prediction (defaults to the model
                     kern). this is useful for examining e.g. subprocesses.
        :returns: (mean, var):
            mean: posterior mean, a Numpy array, Nnew x self.input_dim
            var: posterior variance, a Numpy array, Nnew x 1 if full_cov=False, Nnew x Nnew otherwise

           If full_cov and self.input_dim > 1, the return shape of var is Nnew x Nnew x self.input_dim. If self.input_dim == 1, the return shape is Nnew x Nnew.
           This is to allow for different normalizations of the output dimensions.

        Note: If you want the predictive quantiles (e.g. 95% confidence interval) use :py:func:"~GPy.core.gp.GP.predict_quantiles".
        """
        #predict the latent function values
        mu, var = self._raw_predict(Xnew, full_cov=full_cov, kern=kern)
        if self.normalizer is not None:
            mu, var = self.normalizer.inverse_mean(mu), self.normalizer.inverse_variance(var)

        # now push through likelihood
        mean, var = self.likelihood.predictive_values(mu, var, full_cov, Y_metadata=Y_metadata)
        return mean, var

    def predict_quantiles(self, X, quantiles=(2.5, 97.5), Y_metadata=None, kern=None):
        """
        Get the predictive quantiles around the prediction at X

        :param X: The points at which to make a prediction
        :type X: np.ndarray (Xnew x self.input_dim)
        :param quantiles: tuple of quantiles, default is (2.5, 97.5) which is the 95% interval
        :type quantiles: tuple
        :param kern: optional kernel to use for prediction
        :type predict_kw: dict
        :returns: list of quantiles for each X and predictive quantiles for interval combination
        :rtype: [np.ndarray (Xnew x self.output_dim), np.ndarray (Xnew x self.output_dim)]
        """
        m, v = self._raw_predict(X,  full_cov=False, kern=kern)
        if self.normalizer is not None:
            m, v = self.normalizer.inverse_mean(m), self.normalizer.inverse_variance(v)
        return self.likelihood.predictive_quantiles(m, v, quantiles, Y_metadata=Y_metadata)

    def predictive_gradients(self, Xnew):
        """
        Compute the derivatives of the predicted latent function with respect to X*

        Given a set of points at which to predict X* (size [N*,Q]), compute the
        derivatives of the mean and variance. Resulting arrays are sized:
         dmu_dX* -- [N*, Q ,D], where D is the number of output in this GP (usually one).

        Note that this is not the same as computing the mean and variance of the derivative of the function!

         dv_dX*  -- [N*, Q],    (since all outputs have the same variance)
        :param X: The points at which to get the predictive gradients
        :type X: np.ndarray (Xnew x self.input_dim)
        :returns: dmu_dX, dv_dX
        :rtype: [np.ndarray (N*, Q ,D), np.ndarray (N*,Q) ]

        """
        dmu_dX = np.empty((Xnew.shape[0],Xnew.shape[1],self.output_dim))
        for i in range(self.output_dim):
            dmu_dX[:,:,i] = self.kern.gradients_X(self.posterior.woodbury_vector[:,i:i+1].T, Xnew, self.X)

        # gradients wrt the diagonal part k_{xx}
        dv_dX = self.kern.gradients_X(np.eye(Xnew.shape[0]), Xnew)
        #grads wrt 'Schur' part K_{xf}K_{ff}^{-1}K_{fx}
        alpha = -2.*np.dot(self.kern.K(Xnew, self.X),self.posterior.woodbury_inv)
        dv_dX += self.kern.gradients_X(alpha, Xnew, self.X)
        return dmu_dX, dv_dX

    def predict_jacobian(self, Xnew, kern=None, full_cov=True):
        """
        Compute the derivatives of the posterior of the GP.

        Given a set of points at which to predict X* (size [N*,Q]), compute the
        mean and variance of the derivative. Resulting arrays are sized:

         dL_dX* -- [N*, Q ,D], where D is the number of output in this GP (usually one).
          Note that this is the mean and variance of the derivative,
          not the derivative of the mean and variance! (See predictive_gradients for that)

         dv_dX*  -- [N*, Q],    (since all outputs have the same variance)
          If there is missing data, it is not implemented for now, but
          there will be one output variance per output dimension.

        :param X: The points at which to get the predictive gradients.
        :type X: np.ndarray (Xnew x self.input_dim)
        :param kern: The kernel to compute the jacobian for.
        :param boolean full_cov: whether to return the full covariance of the jacobian.

        :returns: dmu_dX, dv_dX
        :rtype: [np.ndarray (N*, Q ,D), np.ndarray (N*,Q,(D)) ]

        Note: We always return sum in input_dim gradients, as the off-diagonals
        in the input_dim are not needed for further calculations.
        This is a compromise for increase in speed. Mathematically the jacobian would
        have another dimension in Q.
        """
        if kern is None:
            kern = self.kern

        mean_jac = np.empty((Xnew.shape[0],Xnew.shape[1],self.output_dim))

        for i in range(self.output_dim):
            mean_jac[:,:,i] = kern.gradients_X(self.posterior.woodbury_vector[:,i:i+1].T, Xnew, self._predictive_variable)

        dK_dXnew_full = np.empty((self._predictive_variable.shape[0], Xnew.shape[0], Xnew.shape[1]))
        for i in range(self._predictive_variable.shape[0]):
            dK_dXnew_full[i] = kern.gradients_X([[1.]], Xnew, self._predictive_variable[[i]])

        if full_cov:
            dK2_dXdX = kern.gradients_XX([[1.]], Xnew)
        else:
            dK2_dXdX = kern.gradients_XX_diag([[1.]], Xnew)

        def compute_cov_inner(wi):
            if full_cov:
                # full covariance gradients:
                var_jac = dK2_dXdX - np.einsum('qnm,miq->niq', dK_dXnew_full.T.dot(wi), dK_dXnew_full)
            else:
                var_jac = dK2_dXdX - np.einsum('qim,miq->iq', dK_dXnew_full.T.dot(wi), dK_dXnew_full)
            return var_jac

        if self.posterior.woodbury_inv.ndim == 3: # Missing data:
            if full_cov:
                var_jac = np.empty((Xnew.shape[0],Xnew.shape[0],Xnew.shape[1],self.output_dim))
                for d in range(self.posterior.woodbury_inv.shape[2]):
                    var_jac[:, :, :, d] = compute_cov_inner(self.posterior.woodbury_inv[:, :, d])
            else:
                var_jac = np.empty((Xnew.shape[0],Xnew.shape[1],self.output_dim))
                for d in range(self.posterior.woodbury_inv.shape[2]):
                    var_jac[:, :, d] = compute_cov_inner(self.posterior.woodbury_inv[:, :, d])
        else:
            var_jac = compute_cov_inner(self.posterior.woodbury_inv)
        return mean_jac, var_jac

    def predict_wishard_embedding(self, Xnew, kern=None, mean=True, covariance=True):
        """
        Predict the wishard embedding G of the GP. This is the density of the
        input of the GP defined by the probabilistic function mapping f.
        G = J_mean.T*J_mean + output_dim*J_cov.

        :param array-like Xnew: The points at which to evaluate the magnification.
        :param :py:class:`~GPy.kern.Kern` kern: The kernel to use for the magnification.

        Supplying only a part of the learning kernel gives insights into the density
        of the specific kernel part of the input function. E.g. one can see how dense the
        linear part of a kernel is compared to the non-linear part etc.
        """
        if kern is None:
            kern = self.kern

        mu_jac, var_jac = self.predict_jacobian(Xnew, kern, full_cov=False)
        mumuT = np.einsum('iqd,ipd->iqp', mu_jac, mu_jac)
        Sigma = np.zeros(mumuT.shape)
        if var_jac.ndim == 3:
            Sigma[(slice(None), )+np.diag_indices(Xnew.shape[1], 2)] = var_jac.sum(-1)
        else:
            Sigma[(slice(None), )+np.diag_indices(Xnew.shape[1], 2)] = self.output_dim*var_jac
        G = 0.
        if mean:
            G += mumuT
        if covariance:
            G += Sigma
        return G

    def predict_magnification(self, Xnew, kern=None, mean=True, covariance=True):
        """
        Predict the magnification factor as

        sqrt(det(G))

        for each point N in Xnew
        """
        G = self.predict_wishard_embedding(Xnew, kern, mean, covariance)
        from ..util.linalg import jitchol
        mag = np.empty(Xnew.shape[0])
        for n in range(Xnew.shape[0]):
            try:
                mag[n] = np.sqrt(np.exp(2*np.sum(np.log(np.diag(jitchol(G[n, :, :]))))))
            except:
                mag[n] = np.sqrt(np.linalg.det(G[n, :, :]))
        return mag

    def posterior_samples_f(self,X,size=10, full_cov=True):
        """
        Samples the posterior GP at the points X.

        :param X: The points at which to take the samples.
        :type X: np.ndarray (Nnew x self.input_dim)
        :param size: the number of a posteriori samples.
        :type size: int.
        :param full_cov: whether to return the full covariance matrix, or just the diagonal.
        :type full_cov: bool.
        :returns: fsim: set of simulations
        :rtype: np.ndarray (N x samples)
        """
        m, v = self._raw_predict(X,  full_cov=full_cov)
        if self.normalizer is not None:
            m, v = self.normalizer.inverse_mean(m), self.normalizer.inverse_variance(v)
        v = v.reshape(m.size,-1) if len(v.shape)==3 else v
        if not full_cov:
            fsim = np.random.multivariate_normal(m.flatten(), np.diag(v.flatten()), size).T
        else:
            fsim = np.random.multivariate_normal(m.flatten(), v, size).T

        return fsim

    def posterior_samples(self, X, size=10, full_cov=False, Y_metadata=None):
        """
        Samples the posterior GP at the points X.

        :param X: the points at which to take the samples.
        :type X: np.ndarray (Nnew x self.input_dim.)
        :param size: the number of a posteriori samples.
        :type size: int.
        :param full_cov: whether to return the full covariance matrix, or just the diagonal.
        :type full_cov: bool.
        :param noise_model: for mixed noise likelihood, the noise model to use in the samples.
        :type noise_model: integer.
        :returns: Ysim: set of simulations, a Numpy array (N x samples).
        """
        fsim = self.posterior_samples_f(X, size, full_cov=full_cov)
        Ysim = self.likelihood.samples(fsim, Y_metadata=Y_metadata)
        return Ysim

    def plot_f(self, plot_limits=None, which_data_rows='all',
        which_data_ycols='all', fixed_inputs=[],
        levels=20, samples=0, fignum=None, ax=None, resolution=None,
        plot_raw=True,
        linecol=None,fillcol=None, Y_metadata=None, data_symbol='kx',
        apply_link=False):
        """
        Plot the GP's view of the world, where the data is normalized and before applying a likelihood.
        This is a call to plot with plot_raw=True.
        Data will not be plotted in this, as the GP's view of the world
        may live in another space, or units then the data.

        Can plot only part of the data and part of the posterior functions
        using which_data_rowsm which_data_ycols.

        :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits
        :type plot_limits: np.array
        :param which_data_rows: which of the training data to plot (default all)
        :type which_data_rows: 'all' or a slice object to slice model.X, model.Y
        :param which_data_ycols: when the data has several columns (independant outputs), only plot these
        :type which_data_ycols: 'all' or a list of integers
        :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input index i should be set to value v.
        :type fixed_inputs: a list of tuples
        :param resolution: the number of intervals to sample the GP on. Defaults to 200 in 1D and 50 (a 50x50 grid) in 2D
        :type resolution: int
        :param levels: number of levels to plot in a contour plot.
        :param levels: for 2D plotting, the number of contour levels to use is ax is None, create a new figure
        :type levels: int
        :param samples: the number of a posteriori samples to plot
        :type samples: int
        :param fignum: figure to plot on.
        :type fignum: figure number
        :param ax: axes to plot on.
        :type ax: axes handle
        :param linecol: color of line to plot [Tango.colorsHex['darkBlue']]
        :type linecol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param fillcol: color of fill [Tango.colorsHex['lightBlue']]
        :type fillcol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param Y_metadata: additional data associated with Y which may be needed
        :type Y_metadata: dict
        :param data_symbol: symbol as used matplotlib, by default this is a black cross ('kx')
        :type data_symbol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) alongside marker type, as is standard in matplotlib.
        :param apply_link: if there is a link function of the likelihood, plot the link(f*) rather than f*
        :type apply_link: boolean
        """
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from ..plotting.matplot_dep import models_plots
        kw = {}
        if linecol is not None:
            kw['linecol'] = linecol
        if fillcol is not None:
            kw['fillcol'] = fillcol
        return models_plots.plot_fit(self, plot_limits, which_data_rows,
                                     which_data_ycols, fixed_inputs,
                                     levels, samples, fignum, ax, resolution,
                                     plot_raw=plot_raw, Y_metadata=Y_metadata,
                                     data_symbol=data_symbol, apply_link=apply_link, **kw)

    def plot(self, plot_limits=None, which_data_rows='all',
        which_data_ycols='all', fixed_inputs=[],
        levels=20, samples=0, fignum=None, ax=None, resolution=None,
        plot_raw=False, linecol=None,fillcol=None, Y_metadata=None,
        data_symbol='kx', predict_kw=None, plot_training_data=True, samples_y=0, apply_link=False):
        """
        Plot the posterior of the GP.
          - In one dimension, the function is plotted with a shaded region identifying two standard deviations.
          - In two dimsensions, a contour-plot shows the mean predicted function
          - In higher dimensions, use fixed_inputs to plot the GP  with some of the inputs fixed.

        Can plot only part of the data and part of the posterior functions
        using which_data_rowsm which_data_ycols.

        :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits
        :type plot_limits: np.array
        :param which_data_rows: which of the training data to plot (default all)
        :type which_data_rows: 'all' or a slice object to slice model.X, model.Y
        :param which_data_ycols: when the data has several columns (independant outputs), only plot these
        :type which_data_ycols: 'all' or a list of integers
        :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input index i should be set to value v.
        :type fixed_inputs: a list of tuples
        :param resolution: the number of intervals to sample the GP on. Defaults to 200 in 1D and 50 (a 50x50 grid) in 2D
        :type resolution: int
        :param levels: number of levels to plot in a contour plot.
        :param levels: for 2D plotting, the number of contour levels to use is ax is None, create a new figure
        :type levels: int
        :param samples: the number of a posteriori samples to plot, p(f*|y)
        :type samples: int
        :param fignum: figure to plot on.
        :type fignum: figure number
        :param ax: axes to plot on.
        :type ax: axes handle
        :param linecol: color of line to plot [Tango.colorsHex['darkBlue']]
        :type linecol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param fillcol: color of fill [Tango.colorsHex['lightBlue']]
        :type fillcol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param Y_metadata: additional data associated with Y which may be needed
        :type Y_metadata: dict
        :param data_symbol: symbol as used matplotlib, by default this is a black cross ('kx')
        :type data_symbol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) alongside marker type, as is standard in matplotlib.
        :param plot_training_data: whether or not to plot the training points
        :type plot_training_data: boolean
        :param samples_y: the number of a posteriori samples to plot, p(y*|y)
        :type samples_y: int
        :param apply_link: if there is a link function of the likelihood, plot the link(f*) rather than f*, when plotting posterior samples f
        :type apply_link: boolean
        """
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from ..plotting.matplot_dep import models_plots
        kw = {}
        if linecol is not None:
            kw['linecol'] = linecol
        if fillcol is not None:
            kw['fillcol'] = fillcol
        return models_plots.plot_fit(self, plot_limits, which_data_rows,
                                     which_data_ycols, fixed_inputs,
                                     levels, samples, fignum, ax, resolution,
                                     plot_raw=plot_raw, Y_metadata=Y_metadata,
                                     data_symbol=data_symbol, predict_kw=predict_kw,
                                     plot_training_data=plot_training_data, samples_y=samples_y, apply_link=apply_link, **kw)

    def plot_data(self, which_data_rows='all',
        which_data_ycols='all', visible_dims=None,
        fignum=None, ax=None, data_symbol='kx'):
        """
        Plot the training data
          - For higher dimensions than two, use fixed_inputs to plot the data points with some of the inputs fixed.

        Can plot only part of the data
        using which_data_rows and which_data_ycols.

        :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits
        :type plot_limits: np.array
        :param which_data_rows: which of the training data to plot (default all)
        :type which_data_rows: 'all' or a slice object to slice model.X, model.Y
        :param which_data_ycols: when the data has several columns (independant outputs), only plot these
        :type which_data_ycols: 'all' or a list of integers
        :param visible_dims: an array specifying the input dimensions to plot (maximum two)
        :type visible_dims: a numpy array
        :param resolution: the number of intervals to sample the GP on. Defaults to 200 in 1D and 50 (a 50x50 grid) in 2D
        :type resolution: int
        :param levels: number of levels to plot in a contour plot.
        :param levels: for 2D plotting, the number of contour levels to use is ax is None, create a new figure
        :type levels: int
        :param samples: the number of a posteriori samples to plot, p(f*|y)
        :type samples: int
        :param fignum: figure to plot on.
        :type fignum: figure number
        :param ax: axes to plot on.
        :type ax: axes handle
        :param linecol: color of line to plot [Tango.colorsHex['darkBlue']]
        :type linecol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param fillcol: color of fill [Tango.colorsHex['lightBlue']]
        :type fillcol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param data_symbol: symbol as used matplotlib, by default this is a black cross ('kx')
        :type data_symbol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) alongside marker type, as is standard in matplotlib.
        """
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from ..plotting.matplot_dep import models_plots
        kw = {}
        return models_plots.plot_data(self, which_data_rows,
                                     which_data_ycols, visible_dims,
                                     fignum, ax, data_symbol, **kw)

    def errorbars_trainset(self, which_data_rows='all',
            which_data_ycols='all', fixed_inputs=[], fignum=None, ax=None,
            linecol=None, data_symbol='kx', predict_kw=None, plot_training_data=True,lw=None):

        """
        Plot the posterior error bars corresponding to the training data
          - For higher dimensions than two, use fixed_inputs to plot the data points with some of the inputs fixed.

        Can plot only part of the data
        using which_data_rows and which_data_ycols.

        :param which_data_rows: which of the training data to plot (default all)
        :type which_data_rows: 'all' or a slice object to slice model.X, model.Y
        :param which_data_ycols: when the data has several columns (independant outputs), only plot these
        :type which_data_rows: 'all' or a list of integers
        :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input index i should be set to value v.
        :type fixed_inputs: a list of tuples
        :param fignum: figure to plot on.
        :type fignum: figure number
        :param ax: axes to plot on.
        :type ax: axes handle
        :param plot_training_data: whether or not to plot the training points
        :type plot_training_data: boolean
        """
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from ..plotting.matplot_dep import models_plots
        kw = {}
        if lw is not None:
            kw['lw'] = lw
        return models_plots.errorbars_trainset(self, which_data_rows, which_data_ycols, fixed_inputs,
                                    fignum, ax, linecol, data_symbol,
                                    predict_kw, plot_training_data, **kw)

    def plot_magnification(self, labels=None, which_indices=None,
                resolution=50, ax=None, marker='o', s=40,
                fignum=None, legend=True,
                plot_limits=None,
                aspect='auto', updates=False, plot_inducing=True, kern=None, **kwargs):

        import sys
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from ..plotting.matplot_dep import dim_reduction_plots

        return dim_reduction_plots.plot_magnification(self, labels, which_indices,
                resolution, ax, marker, s,
                fignum, plot_inducing, legend,
                plot_limits, aspect, updates, **kwargs)

    def input_sensitivity(self, summarize=True):
        """
        Returns the sensitivity for each dimension of this model
        """
        return self.kern.input_sensitivity(summarize=summarize)

    def optimize(self, optimizer=None, start=None, **kwargs):
        """
        Optimize the model using self.log_likelihood and self.log_likelihood_gradient, as well as self.priors.
        kwargs are passed to the optimizer. They can be:

        :param max_f_eval: maximum number of function evaluations
        :type max_f_eval: int
        :messages: whether to display during optimisation
        :type messages: bool
        :param optimizer: which optimizer to use (defaults to self.preferred optimizer), a range of optimisers can be found in :module:`~GPy.inference.optimization`, they include 'scg', 'lbfgs', 'tnc'.
        :type optimizer: string
        """
        self.inference_method.on_optimization_start()
        try:
            super(GP, self).optimize(optimizer, start, **kwargs)
        except KeyboardInterrupt:
            print("KeyboardInterrupt caught, calling on_optimization_end() to round things up")
            self.inference_method.on_optimization_end()
            raise

    def infer_newX(self, Y_new, optimize=True):
        """
        Infer X for the new observed data *Y_new*.

        :param Y_new: the new observed data for inference
        :type Y_new: numpy.ndarray
        :param optimize: whether to optimize the location of new X (True by default)
        :type optimize: boolean
        :return: a tuple containing the posterior estimation of X and the model that optimize X
        :rtype: (:class:`~GPy.core.parameterization.variational.VariationalPosterior` and numpy.ndarray, :class:`~GPy.core.model.Model`)
        """
        from ..inference.latent_function_inference.inferenceX import infer_newX
        return infer_newX(self, Y_new, optimize=optimize)

    def log_predictive_density(self, x_test, y_test, Y_metadata=None):
        """
        Calculation of the log predictive density

        .. math:
            p(y_{*}|D) = p(y_{*}|f_{*})p(f_{*}|\mu_{*}\\sigma^{2}_{*})

        :param x_test: test locations (x_{*})
        :type x_test: (Nx1) array
        :param y_test: test observations (y_{*})
        :type y_test: (Nx1) array
        :param Y_metadata: metadata associated with the test points
        """
        mu_star, var_star = self._raw_predict(x_test)
        return self.likelihood.log_predictive_density(y_test, mu_star, var_star, Y_metadata=Y_metadata)

    def log_predictive_density_sampling(self, x_test, y_test, Y_metadata=None, num_samples=1000):
        """
        Calculation of the log predictive density by sampling

        .. math:
            p(y_{*}|D) = p(y_{*}|f_{*})p(f_{*}|\mu_{*}\\sigma^{2}_{*})

        :param x_test: test locations (x_{*})
        :type x_test: (Nx1) array
        :param y_test: test observations (y_{*})
        :type y_test: (Nx1) array
        :param Y_metadata: metadata associated with the test points
        :param num_samples: number of samples to use in monte carlo integration
        :type num_samples: int
        """
        mu_star, var_star = self._raw_predict(x_test)
        return self.likelihood.log_predictive_density_sampling(y_test, mu_star, var_star, Y_metadata=Y_metadata, num_samples=num_samples)


## model.py
## Author: Yangfeng Ji
## Date: 09-09-2014
## Time-stamp: <yangfeng 11/05/2014 20:44:25>
## Last changed: umashanthi 11/19/2014

""" As a parsing model, it includes the following functions
1, Mini-batch training on the data generated by the Data class
2, Shift-Reduce RST parsing for a given text sequence
3, Save/load parsing model
"""

from sklearn.svm import LinearSVC
from cPickle import load, dump
from parser import SRParser
from feature import FeatureGenerator
from tree import RSTTree
from util import *
from datastructure import ActionError
import gzip, sys
import numpy as np

class ParsingModel(object):
    def __init__(self, vocab=None, idxlabelmap=None, clf=None):
        """ Initialization

        :type vocab: dict
        :param vocab: mappint from feature templates to feature indices

        :type idxrelamap: dict
        :param idxrelamap: mapping from parsing action indices to
                           parsing actions

        :type clf: LinearSVC
        :param clf: an multiclass classifier from sklearn
        """
        self.vocab = vocab
        # print labelmap
        self.labelmap = idxlabelmap
        if clf is None:
            self.clf = LinearSVC()

    def train(self, trnM, trnL):
        """ Perform batch-learning on parsing model
        """
        self.clf.fit(trnM, trnL)

    def predict(self, features):
        """ Predict parsing actions for a given set
            of features

        :type features: list
        :param features: feature list generated by
                         FeatureGenerator
        """
        vec = vectorize(features, self.vocab)
        predicted_output = self.clf.decision_function(vec)
        idxs = np.argsort(predicted_output[0])[::-1]
        possible_labels = []
        for index in idxs:
            possible_labels.append(self.labelmap[index])
        return possible_labels

    def savemodel(self, fname):
        """ Save model and vocab
        """
        if not fname.endswith('.gz'):
            fname += '.gz'
        D = {'clf':self.clf, 'vocab':self.vocab,
             'idxlabelmap':self.labelmap}
        with gzip.open(fname, 'w') as fout:
            dump(D, fout)
        print 'Save model into file: {}'.format(fname)

    def loadmodel(self, fname):
        """ Load model
        """
        with gzip.open(fname, 'r') as fin:
            D = load(fin)
        self.clf = D['clf']
        self.vocab = D['vocab']
        self.labelmap = D['idxlabelmap']
        print 'Load model from file: {}'.format(fname)

    def sr_parse(self, texts):
        """ Shift-reduce RST parsing based on model prediction

        :type texts: list of string
        :param texts: list of EDUs for parsing
        """
        # Initialize parser
        srparser = SRParser([],[])
        srparser.init(texts)
        # Parsing
        while not srparser.endparsing():
            # Generate features
            stack, queue = srparser.getstatus()
            # Make sure call the generator with
            # same arguments as in data generation part
            fg = FeatureGenerator(stack, queue)
            features = fg.features()
            labels = self.predict(features)
            # Enumerate through all possible actions ranked based on predcition scores
            for i,label in enumerate(labels):
                action = label2action(label)
                try:
                    srparser.operate(action)
                    break # if legal action, end the loop
                except ActionError:
                    if i < len(labels): # if not a legal action, try the next possible action
                        continue
                    else:
                        print "Parsing action error with {}".format(action)
                        sys.exit()

        tree = srparser.getparsetree()
        rst = RSTTree(tree=tree)
        return rst


"""
=============================================================================
Manifold learning on handwritten digits: Locally Linear Embedding, Isomap...
=============================================================================

An illustration of various embeddings on the digits dataset.

The RandomTreesEmbedding, from the :mod:`sklearn.ensemble` module, is not
technically a manifold embedding method, as it learn a high-dimensional
representation on which we apply a dimensionality reduction method.
However, it is often useful to cast a dataset into a representation in
which the classes are linearly-separable.

t-SNE will be initialized with the embedding that is generated by PCA in
this example, which is not the default setting. It ensures global stability
of the embedding, i.e., the embedding does not depend on random
initialization.
"""

# Authors: Fabian Pedregosa <fabian.pedregosa@inria.fr>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Gael Varoquaux
# License: BSD 3 clause (C) INRIA 2011

print(__doc__)
from time import time

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import offsetbox
from sklearn import (manifold, datasets, decomposition, ensemble,
                     discriminant_analysis, random_projection)

digits = datasets.load_digits(n_class=6)
X = digits.data
y = digits.target
n_samples, n_features = X.shape
n_neighbors = 30

#----------------------------------------------------------------------
# Scale and visualize the embedding vectors
def plot_embedding(X, title=None):
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)

    plt.figure()
    ax = plt.subplot(111)
    for i in range(X.shape[0]):
        plt.text(X[i, 0], X[i, 1], str(digits.target[i]),
                 color=plt.cm.Set1(y[i] / 10.),
                 fontdict={'weight': 'bold', 'size': 9})

    if hasattr(offsetbox, 'AnnotationBbox'):
        # only print thumbnails with matplotlib > 1.0
        shown_images = np.array([[1., 1.]])  # just something big
        for i in range(digits.data.shape[0]):
            dist = np.sum((X[i] - shown_images) ** 2, 1)
            if np.min(dist) < 4e-3:
                # don't show points that are too close
                continue
            shown_images = np.r_[shown_images, [X[i]]]
            imagebox = offsetbox.AnnotationBbox(
                offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r),
                X[i])
            ax.add_artist(imagebox)
    plt.xticks([]), plt.yticks([])
    if title is not None:
        plt.title(title)

#----------------------------------------------------------------------
# Plot images of the digits
n_img_per_row = 20
img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row))
for i in range(n_img_per_row):
    ix = 10 * i + 1
    for j in range(n_img_per_row):
        iy = 10 * j + 1
        img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8))

plt.imshow(img, cmap=plt.cm.binary)
plt.xticks([])
plt.yticks([])
plt.title('A selection from the 64-dimensional digits dataset')

#----------------------------------------------------------------------
# Random 2D projection using a random unitary matrix
print("Computing random projection")
rp = random_projection.SparseRandomProjection(n_components=2, random_state=42)
X_projected = rp.fit_transform(X)
plot_embedding(X_projected, "Random Projection of the digits")

#----------------------------------------------------------------------
# Projection on to the first 2 principal components

print("Computing PCA projection")
t0 = time()
X_pca = decomposition.TruncatedSVD(n_components=2).fit_transform(X)
plot_embedding(X_pca,
               "Principal Components projection of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Projection on to the first 2 linear discriminant components

print("Computing Linear Discriminant Analysis projection")
X2 = X.copy()
X2.flat[::X.shape[1] + 1] += 0.01  # Make X invertible
t0 = time()
X_lda = discriminant_analysis.LinearDiscriminantAnalysis(n_components=2).fit_transform(X2, y)
plot_embedding(X_lda,
               "Linear Discriminant projection of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Isomap projection of the digits dataset
print("Computing Isomap embedding")
t0 = time()
X_iso = manifold.Isomap(n_neighbors, n_components=2).fit_transform(X)
print("Done.")
plot_embedding(X_iso,
               "Isomap projection of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Locally linear embedding of the digits dataset
print("Computing LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='standard')
t0 = time()
X_lle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_lle,
               "Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Modified Locally linear embedding of the digits dataset
print("Computing modified LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='modified')
t0 = time()
X_mlle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_mlle,
               "Modified Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# HLLE embedding of the digits dataset
print("Computing Hessian LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='hessian')
t0 = time()
X_hlle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_hlle,
               "Hessian Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# LTSA embedding of the digits dataset
print("Computing LTSA embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='ltsa')
t0 = time()
X_ltsa = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_ltsa,
               "Local Tangent Space Alignment of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# MDS  embedding of the digits dataset
print("Computing MDS embedding")
clf = manifold.MDS(n_components=2, n_init=1, max_iter=100)
t0 = time()
X_mds = clf.fit_transform(X)
print("Done. Stress: %f" % clf.stress_)
plot_embedding(X_mds,
               "MDS embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Random Trees embedding of the digits dataset
print("Computing Totally Random Trees embedding")
hasher = ensemble.RandomTreesEmbedding(n_estimators=200, random_state=0,
                                       max_depth=5)
t0 = time()
X_transformed = hasher.fit_transform(X)
pca = decomposition.TruncatedSVD(n_components=2)
X_reduced = pca.fit_transform(X_transformed)

plot_embedding(X_reduced,
               "Random forest embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Spectral embedding of the digits dataset
print("Computing Spectral embedding")
embedder = manifold.SpectralEmbedding(n_components=2, random_state=0,
                                      eigen_solver="arpack")
t0 = time()
X_se = embedder.fit_transform(X)

plot_embedding(X_se,
               "Spectral embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# t-SNE embedding of the digits dataset
print("Computing t-SNE embedding")
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
t0 = time()
X_tsne = tsne.fit_transform(X)

plot_embedding(X_tsne,
               "t-SNE embedding of the digits (time %.2fs)" %
               (time() - t0))

plt.show()