import os
from collections import defaultdict
import numpy as np

try:
    import matplotlib
    if not os.environ.get('DISPLAY'):
        # Use non-interactive Agg backend
        matplotlib.use('Agg')
    import matplotlib.pyplot as plt
except ImportError:
    import platform
    if platform.python_implementation() == 'PyPy':
        # PyPy doesn't have a version of matplotlib. Make a fake
        # class that raises if it is used. This allows us to use other
        # 'dark' code that happens to import dark.mutations but not use the
        # functions that rely on matplotlib.
        class plt(object):
            def __getattr__(self, _):
                raise NotImplementedError(
                    'matplotlib is not supported under pypy')
    else:
        raise

from random import choice, uniform

from dark import ncbidb

def basePlotter(blastHits, title):
    """
    Plot the reads and the subject, so that bases in the reads which are
    different from the subject are shown. Else a '.' is shown.
    like so:
    subject_gi  ATGCGTACGTACGACACC
    read_1         A......TTC..T

    @param blastHits: A L{dark.blast.BlastHits} instance.
    @param title: A C{str} sequence title that was matched by BLAST. We plot
        the reads that matched this title.
    """
    result = []
    params = blastHits.plotParams
    assert params is not None, ('Oops, it looks like you forgot to run '
                                'computePlotInfo.')

    sequence = ncbidb.getSequence(title, blastHits.records.blastDb)
    subject = sequence.seq
    gi = title.split('|')[1]
    sub = '%s\t \t \t%s' % (gi, subject)
    result.append(sub)

    plotInfo = blastHits.titles[title]['plotInfo']
    assert plotInfo is not None, ('Oops, it looks like you forgot to run '
                                  'computePlotInfo.')

    items = plotInfo['items']
    count = 0
    for item in items:
        count += 1
        hsp = item['hsp']
        queryTitle = blastHits.fasta[item['readNum']].id
        # If the product of the subject and query frame values is +ve,
        # then they're either both +ve or both -ve, so we just use the
        # query as is. Otherwise, we need to reverse complement it.
        if item['frame']['subject'] * item['frame']['query'] > 0:
            query = blastHits.fasta[item['readNum']].seq
            reverse = False
        else:
            # One of the subject or query has negative sense.
            query = blastHits.fasta[
                item['readNum']].reverse_complement().seq
            reverse = True
        query = query.upper()
        queryStart = hsp['queryStart']
        subjectStart = hsp['subjectStart']
        queryEnd = hsp['queryEnd']
        subjectEnd = hsp['subjectEnd']

        # Before comparing the read to the subject, make a string of the
        # same length as the subject, which contains the read and
        # has ' ' where the read does not match.
        # 3 parts need to be taken into account:
        # 1) the left offset (if the query doesn't stick out to the left)
        # 2) the query. if the frame is -1, it has to be reversed.
        # The query consists of 3 parts: left, middle (control for gaps)
        # 3) the right offset

        # Do part 1) and 2).
        if queryStart < 0:
            # The query is sticking out to the left.
            leftQuery = ''
            if subjectStart == 0:
                # The match starts at the first base of the subject.
                middleLeftQuery = ''
            else:
                # The match starts into the subject.
                # Determine the length of the not matching query
                # part to the left.
                leftOffset = -1 * queryStart
                rightOffset = subjectStart + leftOffset
                middleLeftQuery = query[leftOffset:rightOffset]
        else:
            # The query is not sticking out to the left
            # make the left offset.
            leftQuery = queryStart * ' '

            leftQueryOffset = subjectStart - queryStart
            middleLeftQuery = query[:leftQueryOffset]

        # Do part 3).
        # Disregard gaps in subject while adding.
        matchQuery = item['origHsp'].query
        matchSubject = item['origHsp'].sbjct
        index = 0
        mid = ''
        for item in range(len(matchQuery)):
            if matchSubject[index] != ' ':
                mid += matchQuery[index]
            index += 1
        # if the query has been reversed, turn the matched part around
        if reverse:
            rev = ''
            toReverse = mid
            reverseDict = {' ': ' ', '-': '-', 'A': 'T', 'T': 'A',
                           'C': 'G', 'G': 'C', '.': '.', 'N': 'N'}
            for item in toReverse:
                newItem = reverseDict[item]
                rev += newItem
            mid = rev[::-1]

        middleQuery = middleLeftQuery + mid

        # add right not-matching part of the query
        rightQueryOffset = queryEnd - subjectEnd
        rightQuery = query[-rightQueryOffset:]
        middleQuery += rightQuery

        read = leftQuery + middleQuery

        # do part 3)
        offset = len(subject) - len(read)
        # if the read is sticking out to the right
        # chop it off
        if offset < 0:
            read = read[:offset]
        # if it's not sticking out, fill the space with ' '
        elif offset > 0:
            read += offset * ' '

        # compare the subject and the read, make a string
        # called 'comparison', which contains a '.' if the bases
        # are equal and the letter of the read if they are not.
        comparison = ''
        for readBase, subjectBase in zip(read, subject):
            if readBase == ' ':
                comparison += ' '
            elif readBase == subjectBase:
                comparison += '.'
            elif readBase != subjectBase:
                comparison += readBase
            index += 1
        que = '%s \t %s' % (queryTitle, comparison)
        result.append(que)

        # sanity checks
        assert (len(comparison) == len(subject)), (
            '%d != %d' % (len(comparison), len(subject)))

        index = 0
        if comparison[index] == ' ':
            index += 1
        else:
            start = index - 1
            assert (start == queryStart or start == -1), (
                '%s != %s or %s != -1' % (start, queryStart, start))

    return result

def getAPOBECFrequencies(dotAlignment, orig, new, pattern):
    """
    Gets mutation frequencies if they are in a certain pattern.

    @param dotAlignment: result from calling basePlotter
    @param orig: A C{str}, naming the original base
    @param new: A C{str}, what orig was mutated to
    @param pattern: A C{str}m which pattern we're looking for
        (must be one of 'cPattern', 'tPattern')
    """
    cPattern = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG', 'CCT',
                'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC', 'TCG', 'TCT']
    tPattern = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG', 'CTT',
                'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC', 'TTG', 'TTT']
    # choose the right pattern
    if pattern == 'cPattern':
        patterns = cPattern
        middleBase = 'C'
    else:
        patterns = tPattern
        middleBase = 'T'
    # generate the freqs dict with the right pattern
    freqs = defaultdict(int)
    for pattern in patterns:
        freqs[pattern] = 0
    # get the subject sequence from dotAlignment
    subject = dotAlignment[0].split('\t')[3]
    # exclude the subject from the dotAlignment, so just the queries
    # are left over
    queries = dotAlignment[1:]
    for item in queries:
        query = item.split('\t')[1]
        index = 0
        for queryBase in query:
            qBase = query[index]
            sBase = subject[index]
            if qBase == new and sBase == orig:
                try:
                    plusSb = subject[index + 1]
                    minusSb = subject[index - 1]
                except IndexError:
                    plusSb = 'end'
                motif = '%s%s%s' % (minusSb, middleBase, plusSb)
                if motif in freqs:
                    freqs[motif] += 1
            index += 1

    return freqs

def getCompleteFreqs(blastHits):
    """
    Make a dictionary which collects all mutation frequencies from
    all reads.
    Calls basePlotter to get dotAlignment, which is passed to
    getAPOBECFrequencies with the respective parameter, to collect
    the frequencies.

    @param blastHits: A L{dark.blast.BlastHits} instance.
    """
    allFreqs = {}
    for title in blastHits.titles:
        allFreqs[title] = {
            'C>A': {},
            'C>G': {},
            'C>T': {},
            'T>A': {},
            'T>C': {},
            'T>G': {},
        }
        basesPlotted = basePlotter(blastHits, title)
        for mutation in allFreqs[title]:
            orig = mutation[0]
            new = mutation[2]
            if orig == 'C':
                pattern = 'cPattern'
            else:
                pattern = 'tPattern'
            freqs = getAPOBECFrequencies(basesPlotted, orig, new, pattern)
            allFreqs[title][mutation] = freqs
        numberOfReads = len(blastHits.titles[title]['plotInfo']['items'])
        allFreqs[title]['numberOfReads'] = numberOfReads
        allFreqs[title]['bitScoreMax'] = blastHits.titles[
            title]['plotInfo']['bitScoreMax']
    return allFreqs

def makeFrequencyGraph(allFreqs, title, substitution, pattern,
                       color='blue', createFigure=True, showFigure=True,
                       readsAx=False):
    """
    For a title, make a graph showing the frequencies.

    @param allFreqs: result from getCompleteFreqs
    @param title: A C{str}, title of virus of which frequencies should be
        plotted.
    @param substitution: A C{str}, which substitution should be plotted;
        must be one of 'C>A', 'C>G', 'C>T', 'T>A', 'T>C', 'T>G'.
    @param pattern: A C{str}, which pattern we're looking for ( must be
        one of 'cPattern', 'tPattern')
    @param color: A C{str}, color of bars.
    @param createFigure: If C{True}, create a figure.
    @param showFigure: If C{True}, show the created figure.
    @param readsAx: If not None, use this as the subplot for displaying reads.
    """
    cPattern = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG', 'CCT',
                'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC', 'TCG', 'TCT']
    tPattern = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG', 'CTT',
                'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC', 'TTG', 'TTT']

    # choose the right pattern
    if pattern == 'cPattern':
        patterns = cPattern
    else:
        patterns = tPattern

    fig = plt.figure(figsize=(10, 10))
    ax = readsAx or fig.add_subplot(111)
    # how many bars
    N = 16
    ind = np.arange(N)
    width = 0.4
    # make a list in the right order, so that it can be plotted easily
    divisor = allFreqs[title]['numberOfReads']
    toPlot = allFreqs[title][substitution]
    index = 0
    data = []
    for item in patterns:
        newData = toPlot[patterns[index]] / divisor
        data.append(newData)
        index += 1
    # create the bars
    ax.bar(ind, data, width, color=color)
    maxY = np.max(data) + 5
    # axes and labels
    if createFigure:
        title = title.split('|')[4][:50]
        ax.set_title('%s \n %s' % (title, substitution), fontsize=20)
        ax.set_ylim(0, maxY)
        ax.set_ylabel('Absolute Number of Mutations', fontsize=16)
        ax.set_xticks(ind + width)
        ax.set_xticklabels(patterns, rotation=45, fontsize=8)
    if createFigure is False:
        ax.set_xticks(ind + width)
        ax.set_xticklabels(patterns, rotation=45, fontsize=0)
    else:
        if showFigure:
            plt.show()
    return maxY

def makeFrequencyPanel(allFreqs, patientName):
    """
    For a title, make a graph showing the frequencies.

    @param allFreqs: result from getCompleteFreqs
    @param patientName: A C{str}, title for the panel
    """
    titles = sorted(
        iter(allFreqs.keys()),
        key=lambda title: (allFreqs[title]['bitScoreMax'], title))

    origMaxY = 0
    cols = 6
    rows = len(allFreqs)
    figure, ax = plt.subplots(rows, cols, squeeze=False)
    substitutions = ['C>A', 'C>G', 'C>T', 'T>A', 'T>C', 'T>G']
    colors = ['blue', 'black', 'red', 'yellow', 'green', 'orange']

    for i, title in enumerate(titles):
        for index in range(6):
            for subst in allFreqs[str(title)]:
                substitution = substitutions[index]
                print(i, index, title, 'substitution', substitutions[index])
                if substitution[0] == 'C':
                    pattern = 'cPattern'
                else:
                    pattern = 'tPattern'
                maxY = makeFrequencyGraph(allFreqs, title, substitution,
                                          pattern, color=colors[index],
                                          createFigure=False, showFigure=False,
                                          readsAx=ax[i][index])
                if maxY > origMaxY:
                    origMaxY = maxY

            # add title for individual plot.
            # if used for other viruses, this will have to be adapted.
            if index == 0:
                gi = title.split('|')[1]
                titles = title.split(' ')
                try:
                    typeIndex = titles.index('type')
                except ValueError:
                    typeNumber = 'gi: %s' % gi
                else:
                    typeNumber = titles[typeIndex + 1]

                ax[i][index].set_ylabel(('Type %s \n maxBitScore: %s' % (
                    typeNumber, allFreqs[title]['bitScoreMax'])), fontsize=10)
            # add xAxis tick labels
            if i == 0:
                ax[i][index].set_title(substitution, fontsize=13)
            if i == len(allFreqs) - 1 or i == (len(allFreqs) - 1) / 2:
                if index < 3:
                    pat = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG',
                           'CCT', 'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC',
                           'TCG', 'TCT']
                else:
                    pat = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG',
                           'CTT', 'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC',
                           'TTG', 'TTT']
                ax[i][index].set_xticklabels(pat, rotation=45, fontsize=8)

    # make Y-axis equal
    for i, title in enumerate(allFreqs):
        for index in range(6):
            a = ax[i][index]
            a.set_ylim([0, origMaxY])
    # add title of whole panel
    figure.suptitle('Mutation Signatures in %s' % patientName, fontsize=20)
    figure.set_size_inches(5 * cols, 3 * rows, forward=True)
    figure.show()

    return allFreqs

def mutateString(original, n, replacements='acgt'):
    """
    Mutate C{original} in C{n} places with chars chosen from C{replacements}.

    @param original: The original C{str} to mutate.
    @param n: The C{int} number of locations to mutate.
    @param replacements: The C{str} of replacement letters.

    @return: A new C{str} with C{n} places of C{original} mutated.
    @raises ValueError: if C{n} is too high, or C{replacement} contains
        duplicates, or if no replacement can be made at a certain locus
        because C{replacements} is of length one, or if C{original} is of
        zero length.
    """
    if not original:
        raise ValueError('Empty original string passed.')

    if n > len(original):
        raise ValueError('Cannot make %d mutations in a string of length %d' %
                         (n, len(original)))

    if len(replacements) != len(set(replacements)):
        raise ValueError('Replacement string contains duplicates')

    if len(replacements) == 1 and original.find(replacements) != -1:
        raise ValueError('Impossible replacement')

    result = list(original)
    length = len(original)

    for offset in range(length):
        if uniform(0.0, 1.0) < float(n) / (length - offset):
            # Mutate.
            while True:
                new = choice(replacements)
                if new != result[offset]:
                    result[offset] = new
                    break
            n -= 1
            if n == 0:
                break

    return ''.join(result)


#!/usr/bin/env python3
#* This file is part of the MOOSE framework
#* https://www.mooseframework.org
#*
#* All rights reserved, see COPYRIGHT for full restrictions
#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
#*
#* Licensed under LGPL 2.1, please see LICENSE for details
#* https://www.gnu.org/licenses/lgpl-2.1.html

import subprocess
import sys
import csv
import matplotlib.pyplot as plt
import numpy as np

# Use fonts that match LaTeX
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 17
rcParams['font.serif'] = ['Computer Modern Roman']
rcParams['text.usetex'] = True

# Small font size for the legend
from matplotlib.font_manager import FontProperties
fontP = FontProperties()
fontP.set_size('x-small')

def get_last_row(csv_filename):
    '''
    Function which returns just the last row of a CSV file.  We have to
    read every line of the file, there was no stackoverflow example of
    reading just the last line.
    http://stackoverflow.com/questions/20296955/reading-last-row-from-csv-file-python-error
    '''
    with open(csv_filename, 'r') as f:
        lastrow = None
        for row in csv.reader(f):
            if (row != []): # skip blank lines at end of file.
                lastrow = row
        return lastrow

def run_moose(dt, time_integrator):
    '''
    Function which actually runs MOOSE.
    '''
    implicit_flag = 'true'
    explicit_methods = ['ExplicitEuler', 'ExplicitMidpoint', 'Heun', 'Ralston']

    # Set implicit_flag based on TimeIntegrator name
    if (time_integrator in explicit_methods):
        implicit_flag = 'false'

    command_line_args = ['../../../moose_test-opt', '-i', 'scalar.i',
                         'Executioner/dt={}'.format(dt),
                         'Executioner/TimeIntegrator/type={}'.format(time_integrator),
                         'GlobalParams/implicit={}'.format(implicit_flag)]
    try:
        child = subprocess.Popen(command_line_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        # communicate() waits for the process to terminate, so there's no
        # need to wait() for it.  It also sets the returncode attribute on
        # child.
        (stdoutdata, stderrdata) = child.communicate()
        if (child.returncode != 0):
            print('Running MOOSE failed: program output is below:')
            print(stdoutdata)
            raise
    except:
        print('Error executing moose_test')
        sys.exit(1)

    # Parse the last line of the output file to get the error at the final time.
    last_row = get_last_row('scalar_out.csv')
    return float(last_row[1])

#
# Main program
#
fig = plt.figure()
ax1 = fig.add_subplot(111)

# Lists of timesteps and TimeIntegrators to plot.
time_integrators = ['ImplicitEuler', 'ImplicitMidpoint', 'LStableDirk2', 'BDF2', 'CrankNicolson',
                    'LStableDirk3', 'LStableDirk4', 'AStableDirk4',
                    'ExplicitEuler', 'ExplicitMidpoint', 'Heun', 'Ralston']
dts = [.125, .0625, .03125, .015625]

# Plot colors
colors = ['maroon', 'blue', 'green', 'black', 'burlywood', 'olivedrab', 'midnightblue',
          'tomato', 'darkmagenta', 'chocolate', 'lightslategray', 'skyblue']

# Plot line markers
markers = ['v', 'o', 'x', '^', 'H', 'h', '+', 'D', '*', '4', 'd', '8']

# Plot line styles
linestyles = [':', '-', '-.', '--', ':', '-.', '--', ':', '--', '-', '-.', '-']

for i in xrange(len(time_integrators)):
    time_integrator = time_integrators[i]

    # Place to store the results for this TimeIntegrator
    results = []

    # Call MOOSE to compute the results
    for dt in dts:
        results.append(run_moose(dt, time_integrator))

    # Make plot
    xdata = np.log10(np.reciprocal(dts))
    ydata = np.log10(results)

    # Compute linear fit of last three points.
    start_fit = len(xdata) - 3
    end_fit = len(xdata)
    fit = np.polyfit(xdata[start_fit:end_fit], ydata[start_fit:end_fit], 1)

    # Make the plot -- unpack the user's additional plotting arguments
    # from kwargs by prepending with **.
    ax1.plot(xdata, ydata, label=time_integrator + ", $" + "{:.2f}".format(fit[0]) + "$",
             color=colors[i], marker=markers[i], linestyle=linestyles[i])

# Set up the axis labels.
ax1.set_xlabel('$\log (\Delta t^{-1})$')
ax1.set_ylabel('$\log \|e(T)\|_{L^2}$')

# Add a legend
plt.legend(loc='lower left', prop=fontP)

# Save a PDF
plt.savefig('plot.pdf', format='pdf')

# Local Variables:
# python-indent: 2
# End:


import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import random
import datetime

# Import the three change making algorithms
sys.path.insert(0, "../divide-conquer/")
sys.path.insert(0, "../dynamic-programming")
sys.path.insert(0, "../greedy")

from changeslow import changeslow
from changegreedy import changegreedy
from changedp import changedp

### QUESTION 7 ###
def Q7(slow, minChange, maxChange):

	lenV = []
	runtimeGreedy = []
	runtimeDP = []
	runtimeSlow = []

	numExp = 10
	maxRange = 1000
	if (slow):
		maxRange = 10 # some much smaller number

	for i in range(1, maxRange): # V can be of length 1 to (maxRange - 1)
		print "\n------ running V length=" + str(i) + "------"
		lenV.append(i)
		#print "lenV:", lenV
		runtimeGreedy.append(0)
		runtimeDP.append(0)
		runtimeSlow.append(0)

		for j in range(numExp): # run numExp experiments for this length of V
			print "\n  ---- running experiment=" + str(j + 1) + " ----"
			coinArray = []

			for k in range(i): # generate V of size i [1, rand, ..., rand, max=1 + 5*(maxRange-2)]
				if (k == 0):
					coinArray.append(1)
				else:
					randFrom = coinArray[len(coinArray) - 1] + 1
					randTo = coinArray[len(coinArray) - 1] + 5
					coinArray.append(random.randint(randFrom, randTo))
			change = random.randint(minChange, maxChange)
			#print "  coinArray:", coinArray
			#print "  change:", change

			print "  running greedy..."
			start = datetime.datetime.now()
			_, _ = changegreedy(coinArray, change)
			end = datetime.datetime.now()
			delta = end - start
			delta = int(delta.total_seconds() * 1000000)
			print "    " + str(delta)
			runtimeGreedy[i - 1] += delta

			print "  running DP..."
			start = datetime.datetime.now()
			_, _ = changedp(coinArray, change)
			end = datetime.datetime.now()
			delta = end - start
			delta = int(delta.total_seconds() * 1000000)
			print "    " + str(delta)
			runtimeDP[i - 1] += delta

			if (slow):
				print "  running slow..."
				start = datetime.datetime.now()
				_, _ = changeslow(coinArray, change)
				end = datetime.datetime.now()
				delta = end - start
				delta = int(delta.total_seconds() * 1000000)
				print "    " + str(delta)
				runtimeSlow[i - 1] += delta

		runtimeGreedy[i - 1] /= numExp
		runtimeDP[i - 1] /= numExp
		if (slow):
			runtimeSlow[i - 1] /= numExp

	plt.figure(21)
	plt.plot(lenV, runtimeGreedy, 'b-', linewidth=2.0, label='Greedy')
	plt.plot(lenV, runtimeDP, 'r--', linewidth=2.0, label='DP')
	if (slow):
		plt.plot(lenV, runtimeSlow, 'g-.', linewidth=2.0, label='Slow')
	plt.legend(loc='upper left')
	plt.title('Runtime vs len(V[]) for randomized V[] and A')
	plt.ylabel('Avg. Runtime (10^-6 sec)')
	plt.xlabel('len(V[])')
	plt.grid(True)
	if (slow):
		plt.savefig('img/Q7slow_runtime.png', bbox_inches='tight')
	else:
		plt.savefig('img/Q7_runtime.png', bbox_inches='tight')

def main():
	Q7(False, 100, 100)
	#Q7(True)

if __name__ == "__main__":
	main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# from scipy import stats
# import statsmodels.api as sm
# from numpy.random import randn
import matplotlib as mpl
# import seaborn as sns

# sns.set_color_palette("deep", desat=.6)
mpl.rc("figure", figsize=(8, 4))

def Compavg():
	data=Total()
	markMax=[]
	markAvg=[]
	N = 5
	ind = np.arange(N)
	width = 0.35
	fig = plt.figure()
	ax = fig.add_subplot(111)
	markMax.extend((data["Total"].max(),data["Total.1"].max(),data["Total.2"].max(),data["Total.3"].max(),data["Total.4"].max()))
	markAvg.extend((data["Total"].mean(),data["Total.1"].mean(),data["Total.2"].mean(),data["Total.3"].mean(),data["Total.4"].mean()))
	rects1 = ax.bar(ind, markMax, width, color='black')
	rects2 = ax.bar(ind+width, markAvg, width, color='green')
	ax.set_xlim(-width,len(ind)+width)
	ax.set_ylim(0,120)
	ax.set_ylabel('Marks')
	ax.set_title('Max, Mean and Your Marks')
	xTickMarks = ['Subject'+str(i) for i in range(1,6)]
	ax.set_xticks(ind+width)
	xtickNames = ax.set_xticklabels(xTickMarks)
	plt.setp(xtickNames, rotation=10, fontsize=10)
	ax.legend( (rects1[0], rects2[0]), ('Max', 'Mean') )
	plt.show()

def compSub():
	# max_data = np.r_[data["Total"]].max()
	# bins = np.linspace(0, max_data, max_data + 1)
	data=Total()
	plt.hist(data['Total'],linewidth=0, alpha=.7)
	plt.hist(data['Total.1'],linewidth=0,alpha=.7)
	plt.hist(data['Total.2'],linewidth=0,alpha=.7)
	plt.hist(data['Total.3'],linewidth=0,alpha=.7)
	plt.hist(data['Total.4'],linewidth=0,alpha=.7)
	plt.title("Total marks Histogram")
	plt.xlabel("Value")
	plt.ylabel("Frequency")
	plt.show()

def Total():
	data=pd.read_csv("output10cs.csv")
	df3=data[['Total','Total.1','Total.2','Total.3','Total.4','Total.5','Total.6','Total.7']]
	data["Main Total"]=df3.sum(axis=1)
	data = data.dropna()
	data.reset_index(drop=True)
	return data

#compSub()
# Compavg()


"""
==========================================
One-class SVM with non-linear kernel (RBF)
==========================================

An example using a one-class SVM for novelty detection.

:ref:`One-class SVM <svm_outlier_detection>` is an unsupervised
algorithm that learns a decision function for novelty detection:
classifying new data as similar or different to the training set.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.font_manager
from sklearn import svm

xx, yy = np.meshgrid(np.linspace(-5, 5, 500), np.linspace(-5, 5, 500))
# Generate train data
X = 0.3 * np.random.randn(100, 2)
X_train = np.r_[X + 2, X - 2]
# Generate some regular novel observations
X = 0.3 * np.random.randn(20, 2)
X_test = np.r_[X + 2, X - 2]
# Generate some abnormal novel observations
X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))

# fit the model
clf = svm.OneClassSVM(nu=0.1, kernel="rbf", gamma=0.1)
clf.fit(X_train)
y_pred_train = clf.predict(X_train)
y_pred_test = clf.predict(X_test)
y_pred_outliers = clf.predict(X_outliers)
n_error_train = y_pred_train[y_pred_train == -1].size
n_error_test = y_pred_test[y_pred_test == -1].size
n_error_outliers = y_pred_outliers[y_pred_outliers == 1].size

# plot the line, the points, and the nearest vectors to the plane
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)

plt.title("Novelty Detection")
plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), 0, 7), cmap=plt.cm.Blues_r)
a = plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors='red')
plt.contourf(xx, yy, Z, levels=[0, Z.max()], colors='orange')

b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white')
b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='green')
c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='red')
plt.axis('tight')
plt.xlim((-5, 5))
plt.ylim((-5, 5))
plt.legend([a.collections[0], b1, b2, c],
           ["learned frontier", "training observations",
            "new regular observations", "new abnormal observations"],
           loc="upper left",
           prop=matplotlib.font_manager.FontProperties(size=11))
plt.xlabel(
    "error train: %d/200 ; errors novel regular: %d/40 ; "
    "errors novel abnormal: %d/40"
    % (n_error_train, n_error_test, n_error_outliers))
plt.show()


# Python script to visualize rotation about a non-body axis.

# Let the lab frame be the inertial frame S.
# Let the origin of the rigid body be O, in the inertial frame S'.
# Let r_ss' be the vector from S to S'.

# Let the body frame relative to O be S''.
# Consider a fixed point on the body, r_s' in S', and r_s'' in S''.
# Assume the body is subject to zero external torques.
# It must be rotating about a fixed axis, n, by Euler's rotation theorem.
# It must have a constant angular velocity about that axis by d/dt L = sum(T_external) = 0 and L = Jw about the rotation axis.

# Let R be the rotation matrix mapping a vector in S'' to S', with inverse R^T
# We know r_s' = R r_s''
# We know d/dt r_s' = (dR/dt R^T) * (R r_s'') = (dR/dt R^T) r_s'
# Therefore we expect (dR/dt R^T) to be the operator (w x) in the S' frame.
# The goal of this script is to visualize this.

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import sys

import numpy as np
from numpy import pi as pi
from numpy import cos as c
from numpy import sin as s
from numpy import dot as dot
from numpy import transpose as transpose

# The axis phi is a rotation about the z axis in the body frame (yaw)
# The axis theta is a rotation about the y axis in the phi-rotated body frame (pitch)
# The axis psi is a rotation about the x axis in the phi, theta-rotated body frame (roll)
def R(phi, theta, psi):
  R = np.zeros((3,3))

  R[0,0] = c(phi)*c(theta)
  R[1,0] = s(phi)*c(theta)
  R[2,0] = -s(theta)

  R[0,1] = -s(phi)*c(psi) + c(phi)*s(theta)*s(psi)
  R[1,1] = c(phi)*c(psi) + s(phi)*s(theta)*s(psi)
  R[2,1] = c(theta)*s(psi)

  R[0,2] = s(phi)*s(psi) + c(phi)*s(theta)*c(psi)
  R[1,2] = -c(phi)*s(psi) + s(phi)*s(theta)*c(psi)
  R[2,2] = c(theta)*c(psi)

  return R

# Rotate z-axis (0,0,1) by pi radians about x-axis. Should end up at (0,0,-1) cutting across y.
# Rotate (0,0,-1) by pi radians about y-axis. Should end up at (0,0,1) again, cutting across x.
# Try both at the same time. Should still end up at (0,0,1).
def test_R():
  e3_spp = np.array((0,0,1))
  vectors = []
  for k in np.linspace(0,pi,100):
    vectors.append(dot(R(0,0,k), e3_spp))
  e3_spp = vectors[-1]
  for k in np.linspace(0,pi,100):
    vectors.append(dot(R(0,k,0), e3_spp))
  e3_spp = vectors[-1]
  for k in np.linspace(0,pi,100):
    vectors.append(dot(R(0,k,k), e3_spp))

  xs = [k[0] for k in vectors]
  ys = [k[1] for k in vectors]
  zs = [k[2] for k in vectors]

  fig = plt.figure()
  ax = fig.add_subplot(111, projection='3d')
  ax.plot(xs=xs,ys=ys,zs=zs)
  ax.set_xlabel("x")
  ax.set_ylabel("y")
  ax.set_zlabel("z")
  plt.show()

# Sets values lower than epsilon to zero.
# Prints the result with precision 0.3f.
def sanitize_matrix(A):
  print ""
  epsilon = 0.001
  for r in xrange(3):
    text = ""
    for c in xrange(3):
      if abs(A[r, c]) < epsilon:
        A[r,c] = 0
      text += "%6.2f,\t" % A[r,c]
    print text[:-2]
  print ""

def sanitize_vector(a):
  print ""
  epsilon = 0.001
  text = ""
  for r in xrange(3):
    if abs(a[r]) < epsilon:
      a[r] = 0
    text += "%6.2f,\t" % a[r]
  print text[:-2]
  print ""

def vectorize(W):
  v = np.zeros(3)
  v[0] = W[1,0]
  v[1] = W[0,2]
  v[2] = W[2,1]
  return v

# This is the (w x) operator, W, with respect to changing body yaw, pitch, and roll.
# It is dR/dt R^T. The arguments are the current Euler angles and their time derivatives.
def W(phi, theta, psi, dphi, dtheta, dpsi):
  Rp = np.zeros((3,3))

  Rp[0,0] = (-s(phi)*dphi)*c(theta)
  Rp[0,0] += c(phi)*(-s(theta)*dtheta)

  Rp[1,0] = (c(phi)*dphi)*c(theta)
  Rp[1,0] += s(phi)*(-s(theta)*dtheta)

  Rp[2,0] = -c(theta)*dtheta

  Rp[0,1] = (-c(phi)*dphi)*c(psi)
  Rp[0,1] += -s(phi)*(-s(psi)*dpsi)
  Rp[0,1] += (-s(phi)*dphi)*s(theta)*s(psi)
  Rp[0,1] += c(phi)*(c(theta)*dtheta)*s(psi)
  Rp[0,1] += c(phi)*s(theta)*(c(psi)*dpsi)

  Rp[1,1] = (-s(phi)*dphi)*c(psi)
  Rp[1,1] += c(phi)*(-s(psi)*dpsi)
  Rp[1,1] += (c(phi)*dphi)*s(theta)*s(psi)
  Rp[1,1] += s(phi)*(c(theta)*dtheta)*s(psi)
  Rp[1,1] += s(phi)*s(theta)*(c(psi)*dpsi)

  Rp[2,1] = (-s(theta)*dtheta)*s(psi)
  Rp[2,1] += c(theta)*(c(psi)*dpsi)

  Rp[0,2] = (c(phi)*dphi)*s(psi)
  Rp[0,2] += s(phi)*(c(psi)*dpsi)
  Rp[0,2] += (-s(phi)*dphi)*s(theta)*c(psi)
  Rp[0,2] += c(phi)*(c(theta)*dtheta)*c(psi)
  Rp[0,2] += c(phi)*s(theta)*(-s(psi)*dpsi)

  Rp[1,2] = (s(phi)*dphi)*s(psi)
  Rp[1,2] += -c(phi)*(c(psi)*dpsi)
  Rp[1,2] += (c(phi)*dphi)*s(theta)*c(psi)
  Rp[1,2] += s(phi)*(c(theta)*dtheta)*c(psi)
  Rp[1,2] += s(phi)*s(theta)*(-s(psi)*dpsi)

  Rp[2,2] = (-s(theta)*dtheta)*c(psi)
  Rp[2,2] += c(theta)*(-s(psi)*dpsi)

  w_i = vectorize(dot(Rp, transpose(R(phi,theta,psi))))
  w_b = dot(transpose(R(phi,theta,psi)), w_i)

  return (w_i, w_b)

def test_W():
  # Is the effective w for a rotation of x rad/s about ek just.. ek*x,
  # regardless of the angle about axis ek? We expect W = -W^T as well.
  # sanitize_matrix(W(3*pi/12,0,0,2*pi,0,0)[0])
  # sanitize_matrix(W(0,3*pi/12,0,0,2*pi,0)[0])
  # sanitize_matrix(W(0,0,3*pi/12,0,0,2*pi)[0])

  # Let's see what it looks like once we've rotated a bit.
  # It's still skew antisymmetric with zero trace! This looks like the operation (w x)!!!!
  phi, theta, psi = (pi/4, 3*pi/12, -pi)
  w_i, w_b = W(phi, theta, psi, pi, 2*pi, 3*pi)

def Main():
  test_W()

if __name__ == "__main__":
  Main()


# Display a list of *.dat files in a bar chart.
# Based on an example from https://chrisalbon.com/python/matplotlib_grouped_bar_plot.html

import sys
import os
import matplotlib.pyplot as plt
import numpy as np

if len(sys.argv) > 3 and (len(sys.argv)-3) % 2 :
    print("usage: python3 {0} <benchmark> <image path> (<dat1> <legend1> [<dat2> <legend2>] .. [<datN> <legendN>] ) ".format(os.path.basename(sys.argv[0])))
    exit(1)

benchmark  = sys.argv[1]
image_path = sys.argv[2]
groups     = (len(sys.argv)-3)/2

# Load benchark
domains = ()
nb_setups = 0
for line in open(benchmark,'r'):
    n, snx, sny, snz = line.split()
    domains += ( r"{0}$^3$".format(snx), ) #+= ( "{0}x{1}x{2}".format(snx, sny, snz), )
    nb_setups += 1

# Setting the positions and width for the bars
pos = list(range(nb_setups))
width = 1 / (groups+2)

# Plotting the bars
fig, ax = plt.subplots(figsize=(10,5))

prop_iter = iter(plt.rcParams['axes.prop_cycle'])

legends = ()
maxLups = 0
for i, argi in enumerate(range(3, len(sys.argv), 2)):
    mlups = np.array(list(map(float, open(sys.argv[argi])))) / 1E6
    legends += ( sys.argv[argi+1], )
    maxLups = max(maxLups, max(mlups))
    plt.bar([p + width*i for p in pos],
            mlups,
            width,
            alpha=0.5,
            color=next(prop_iter)['color'])

# Set the y axis label
ax.set_ylabel('MLUPS')
ax.set_xlabel('Taille du sous-domaine')

# Set the chart's title
#ax.set_title(title)

# Set the position of the x ticks
ax.set_xticks([p + 1.5 * width for p in pos])

# Set the labels for the x ticks
ax.set_xticklabels(domains)

# Setting the x-axis and y-axis limits
plt.xlim(min(pos)-width, max(pos)+width*4)
#plt.ylim([0, maxLups] )

# Adding the legend and showing the plot
plt.legend(legends, loc='upper center')
ax.yaxis.grid()
plt.savefig(image_path)
plt.tight_layout()
plt.show()


import numpy as np
from scipy.sparse import csr_matrix

from sklearn.utils.testing import assert_array_equal, assert_equal, assert_true
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_array_almost_equal, assert_raises
from sklearn.utils.testing import assert_less_equal

from sklearn.metrics.pairwise import kernel_metrics
from sklearn.kernel_approximation import RBFSampler
from sklearn.kernel_approximation import AdditiveChi2Sampler
from sklearn.kernel_approximation import SkewedChi2Sampler
from sklearn.kernel_approximation import Nystroem
from sklearn.metrics.pairwise import polynomial_kernel, rbf_kernel

# generate data
rng = np.random.RandomState(0)
X = rng.random_sample(size=(300, 50))
Y = rng.random_sample(size=(300, 50))
X /= X.sum(axis=1)[:, np.newaxis]
Y /= Y.sum(axis=1)[:, np.newaxis]

def test_additive_chi2_sampler():
    # test that AdditiveChi2Sampler approximates kernel on random data

    # compute exact kernel
    # appreviations for easier formular
    X_ = X[:, np.newaxis, :]
    Y_ = Y[np.newaxis, :, :]

    large_kernel = 2 * X_ * Y_ / (X_ + Y_)

    # reduce to n_samples_x x n_samples_y by summing over features
    kernel = (large_kernel.sum(axis=2))

    # approximate kernel mapping
    transform = AdditiveChi2Sampler(sample_steps=3)
    X_trans = transform.fit_transform(X)
    Y_trans = transform.transform(Y)

    kernel_approx = np.dot(X_trans, Y_trans.T)

    assert_array_almost_equal(kernel, kernel_approx, 1)

    X_sp_trans = transform.fit_transform(csr_matrix(X))
    Y_sp_trans = transform.transform(csr_matrix(Y))

    assert_array_equal(X_trans, X_sp_trans.A)
    assert_array_equal(Y_trans, Y_sp_trans.A)

    # test error is raised on negative input
    Y_neg = Y.copy()
    Y_neg[0, 0] = -1
    assert_raises(ValueError, transform.transform, Y_neg)

    # test error on invalid sample_steps
    transform = AdditiveChi2Sampler(sample_steps=4)
    assert_raises(ValueError, transform.fit, X)

    # test that the sample interval is set correctly
    sample_steps_available = [1, 2, 3]
    for sample_steps in sample_steps_available:

        # test that the sample_interval is initialized correctly
        transform = AdditiveChi2Sampler(sample_steps=sample_steps)
        assert_equal(transform.sample_interval, None)

        # test that the sample_interval is changed in the fit method
        transform.fit(X)
        assert_not_equal(transform.sample_interval_, None)

    # test that the sample_interval is set correctly
    sample_interval = 0.3
    transform = AdditiveChi2Sampler(sample_steps=4,
                                    sample_interval=sample_interval)
    assert_equal(transform.sample_interval, sample_interval)
    transform.fit(X)
    assert_equal(transform.sample_interval_, sample_interval)

def test_skewed_chi2_sampler():
    # test that RBFSampler approximates kernel on random data

    # compute exact kernel
    c = 0.03
    # appreviations for easier formular
    X_c = (X + c)[:, np.newaxis, :]
    Y_c = (Y + c)[np.newaxis, :, :]

    # we do it in log-space in the hope that it's more stable
    # this array is n_samples_x x n_samples_y big x n_features
    log_kernel = ((np.log(X_c) / 2.) + (np.log(Y_c) / 2.) + np.log(2.) -
                  np.log(X_c + Y_c))
    # reduce to n_samples_x x n_samples_y by summing over features in log-space
    kernel = np.exp(log_kernel.sum(axis=2))

    # approximate kernel mapping
    transform = SkewedChi2Sampler(skewedness=c, n_components=1000,
                                  random_state=42)
    X_trans = transform.fit_transform(X)
    Y_trans = transform.transform(Y)

    kernel_approx = np.dot(X_trans, Y_trans.T)
    assert_array_almost_equal(kernel, kernel_approx, 1)

    # test error is raised on negative input
    Y_neg = Y.copy()
    Y_neg[0, 0] = -1
    assert_raises(ValueError, transform.transform, Y_neg)

def test_rbf_sampler():
    # test that RBFSampler approximates kernel on random data
    # compute exact kernel
    gamma = 10.
    kernel = rbf_kernel(X, Y, gamma=gamma)

    # approximate kernel mapping
    rbf_transform = RBFSampler(gamma=gamma, n_components=1000, random_state=42)
    X_trans = rbf_transform.fit_transform(X)
    Y_trans = rbf_transform.transform(Y)
    kernel_approx = np.dot(X_trans, Y_trans.T)

    error = kernel - kernel_approx
    assert_less_equal(np.abs(np.mean(error)), 0.01)  # close to unbiased
    np.abs(error, out=error)
    assert_less_equal(np.max(error), 0.1)  # nothing too far off
    assert_less_equal(np.mean(error), 0.05)  # mean is fairly close

def test_input_validation():
    # Regression test: kernel approx. transformers should work on lists
    # No assertions; the old versions would simply crash
    X = [[1, 2], [3, 4], [5, 6]]
    AdditiveChi2Sampler().fit(X).transform(X)
    SkewedChi2Sampler().fit(X).transform(X)
    RBFSampler().fit(X).transform(X)

    X = csr_matrix(X)
    RBFSampler().fit(X).transform(X)

def test_nystroem_approximation():
    # some basic tests
    rnd = np.random.RandomState(0)
    X = rnd.uniform(size=(10, 4))

    # With n_components = n_samples this is exact
    X_transformed = Nystroem(n_components=X.shape[0]).fit_transform(X)
    K = rbf_kernel(X)
    assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)

    trans = Nystroem(n_components=2, random_state=rnd)
    X_transformed = trans.fit(X).transform(X)
    assert_equal(X_transformed.shape, (X.shape[0], 2))

    # test callable kernel
    linear_kernel = lambda X, Y: np.dot(X, Y.T)
    trans = Nystroem(n_components=2, kernel=linear_kernel, random_state=rnd)
    X_transformed = trans.fit(X).transform(X)
    assert_equal(X_transformed.shape, (X.shape[0], 2))

    # test that available kernels fit and transform
    kernels_available = kernel_metrics()
    for kern in kernels_available:
        trans = Nystroem(n_components=2, kernel=kern, random_state=rnd)
        X_transformed = trans.fit(X).transform(X)
        assert_equal(X_transformed.shape, (X.shape[0], 2))

def test_nystroem_singular_kernel():
    # test that nystroem works with singular kernel matrix
    rng = np.random.RandomState(0)
    X = rng.rand(10, 20)
    X = np.vstack([X] * 2)  # duplicate samples

    gamma = 100
    N = Nystroem(gamma=gamma, n_components=X.shape[0]).fit(X)
    X_transformed = N.transform(X)

    K = rbf_kernel(X, gamma=gamma)

    assert_array_almost_equal(K, np.dot(X_transformed, X_transformed.T))
    assert_true(np.all(np.isfinite(Y)))

def test_nystroem_poly_kernel_params():
    # Non-regression: Nystroem should pass other parameters beside gamma.
    rnd = np.random.RandomState(37)
    X = rnd.uniform(size=(10, 4))

    K = polynomial_kernel(X, degree=3.1, coef0=.1)
    nystroem = Nystroem(kernel="polynomial", n_components=X.shape[0],
                        degree=3.1, coef0=.1)
    X_transformed = nystroem.fit_transform(X)
    assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)

def test_nystroem_callable():
    # Test Nystroem on a callable.
    rnd = np.random.RandomState(42)
    n_samples = 10
    X = rnd.uniform(size=(n_samples, 4))

    def logging_histogram_kernel(x, y, log):
        """Histogram kernel that writes to a log."""
        log.append(1)
        return np.minimum(x, y).sum()

    kernel_log = []
    X = list(X)     # test input validation
    Nystroem(kernel=logging_histogram_kernel,
             n_components=(n_samples - 1),
             kernel_params={'log': kernel_log}).fit(X)
    assert_equal(len(kernel_log), n_samples * (n_samples - 1) / 2)


"""
This module contains the logic to run the simulation.
"""
import sys
import os
import argparse
import numpy as np
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from robot_localisation.grid import Grid, build_transition_matrix
from robot_localisation.robot import Robot, Sensor
from robot_localisation.hmm_filter import FilterState

def help_text():
    """
    Return a helpful text explaining usage of the program.
    """
    return """
------------------------------- HMM Filtering ---------------------------------
Type a command to get started. Type 'quit' or 'q' to quit.

Valid commands (all commands are case insensitive):
    ENTER                   move the robot one step further in the simulation,
                            will also output current pose and estimated
                            position of the robot
    help                    show this help text
    show T                  show the transition matrix T
    show f                  show the filter column vector
    show O                  show the observation matrix
    quit | q                quit the program
-------------------------------------------------------------------------------
    """

def main():
    parser = argparse.ArgumentParser(description='Robot localisation with HMM')
    parser.add_argument(
        '-r', '--rows',
        type=int,
        help='the number of rows on the grid, default is 4',
        default=4)
    parser.add_argument(
        '-c', '--columns',
        type=int,
        help='the number of columns on the grid, default is 4',
        default=4)
    args = parser.parse_args()

    # Initialise the program
    size = (args.rows, args.columns)
    the_T_matrix = build_transition_matrix(*size)
    the_filter = FilterState(transition=the_T_matrix)
    the_sensor = Sensor()
    the_grid = Grid(*size)
    the_robot = Robot(the_grid, the_T_matrix)
    sensor_value = None
    obs = None

    print(help_text())
    print("Grid size is {} x {}".format(size[0], size[1]))
    print(the_robot)
    print("The sensor says: {}".format(sensor_value))
    filter_est = the_grid.index_to_pose(the_filter.belief_state)
    pos_est = (filter_est[0], filter_est[1])
    print("The HMM filter thinks the robot is at {}".format(filter_est))
    print("The Manhattan distance is: {}".format(
        manhattan(the_robot.get_position(), pos_est)))
    np.set_printoptions(linewidth=1000)

    # Main loop
    while True:
        user_command = str(input('> '))
        if user_command.upper() == 'QUIT' or user_command.upper() == 'Q':
            break
        elif user_command.upper() == 'HELP':
            print(help_text())
        elif user_command.upper() == 'SHOW T':
            print(the_T_matrix)
        elif user_command.upper() == 'SHOW F':
            print(the_filter.belief_matrix)
        elif user_command.upper() == 'SHOW O':
            print(obs)
        elif not user_command:
            # take a step then approximate etc.
            the_robot.step()
            sensor_value = the_sensor.get_position(the_robot)
            obs = the_sensor.get_obs_matrix(sensor_value, size)
            the_filter.forward(obs)

            print(the_robot)
            print("The sensor says: {}".format(sensor_value))
            filter_est = the_grid.index_to_pose(the_filter.belief_state)
            pos_est = (filter_est[0], filter_est[1])
            print("The HMM filter thinks the robot is at {}".format(filter_est))
            print("The Manhattan distance is: {}".format(
                manhattan(the_robot.get_position(), pos_est)))

        else:
            print("Unknown command!")

def manhattan(pos1, pos2):
    """
    Calculate the Manhattan distance between pos1 and pos2.
    """
    x1, y1 = pos1
    x2, y2 = pos2
    return abs(x1-x2) + abs(y1-y2)

def automated_run():
    import matplotlib.pyplot as plt

    fig = plt.figure(figsize=(10, 7))
    navg = 20
    nsteps = 10

    for size in (2, 2), (3, 3), (4, 4), (5, 5), (10, 10):

        avg_distances = np.zeros(shape=(nsteps+1,))

        for n in range(navg):

            distances = list()
            none_values = list()

            the_T_matrix = build_transition_matrix(*size)
            the_filter = FilterState(transition=the_T_matrix)
            the_sensor = Sensor()
            the_grid = Grid(*size)
            the_robot = Robot(the_grid, the_T_matrix)

            # get the manhattan distance at the start
            filter_est = the_grid.index_to_pose(the_filter.belief_state)
            pos_est = (filter_est[0], filter_est[1])
            distances.append(manhattan(the_robot.get_position(), pos_est))

            for i in range(nsteps):

                # take a step then approximate etc.
                the_robot.step()
                sensor_value = the_sensor.get_position(the_robot)
                if sensor_value is None:
                    none_values.append(i)  # keep track of where None was returned

                obs = the_sensor.get_obs_matrix(sensor_value, size)
                the_filter.forward(obs)

                filter_est = the_grid.index_to_pose(the_filter.belief_state)
                pos_est = (filter_est[0], filter_est[1])
                distances.append(manhattan(the_robot.get_position(), pos_est))

            avg_distances += np.array(distances)

        avg_distances /= navg
        base_line, = plt.plot(avg_distances, label="Grid size {}".format(size))

        # for point in none_values:
        #     plt.scatter(point, distances[point], marker='o',
        #                 color=base_line.get_color(), s=40)

    plt.legend()
    plt.xlim(0, nsteps)
    plt.ylim(0,)
    plt.ylabel("Manhattan distance")
    plt.xlabel("Steps")
    plt.title("Manhattan distance from true position and inferred position \n"
              "from the hidden Markov model (average over %s runs)" % navg)
    fig.savefig("automated_run.png")
    plt.show()

if __name__ == '__main__':
    main()
    # automated_run()


from __future__ import absolute_import, print_function

import sys, os
from six.moves.urllib import request as urllib2
from six.moves import cStringIO as StringIO
import pandas as pd

try:
    import click
    is_click = True
except ImportError:
    is_click = False

from . import help_messages as hm
from .utils import (get_chart_params, get_charts_mapping,
                    get_data_series, keep_source_input_sync, get_data_from_url)
from .. import charts as bc
from ..charts import utils as bc_utils
from bokeh.models.widgets import Button

# Define a mapping to connect chart types supported arguments and chart classes
CHARTS_MAP = get_charts_mapping()

if is_click:
    @click.command()
    @click.option('--input', 'input_source', default=None,help=hm.HELP_INPUT)
    @click.option('--output', default='file://cli_output.html', help=hm.HELP_OUTPUT)
    @click.option('--title', default='Bokeh CLI')
    @click.option('--chart_type', default='Line')
    @click.option('--index', default='', help=hm.HELP_INDEX)
    @click.option('--series', default='', help=hm.HELP_SERIES)
    @click.option('--palette')
    @click.option('--buffer', default='f', help=hm.HELP_BUFFER)
    @click.option('--sync_with_source', default=False)
    @click.option('--update_ranges', 'update_ranges', flag_value=True,
                  default=False)
    @click.option('--legend', 'show_legend', flag_value=True,
                  default=False)
    @click.option('--window_size', default='0', help=hm.HELP_WIN_SIZE)
    @click.option('--map', 'map_', default=None)
    @click.option('--map_zoom', 'map_zoom', default=12)
    @click.option('--map_layer', 'map_layer', default="hybrid")
    @click.option('--smart_filters', 'smart_filters', flag_value=True,
                  default=False)
    def cli(input_source, output, title, chart_type, series, palette, index,
            buffer, sync_with_source, update_ranges, show_legend, window_size,
            map_, smart_filters, map_zoom, map_layer):
        """Bokeh Command Line Tool is a minimal client to access high level plotting
        functionality provided by bokeh.charts API.

        Examples:

        >> python bokeh-cli.py --title "My Nice Plot" --series "High,Low,Close"
        --chart_type "Line" --palette Reds --input sample_data/stocks_data.csv

        >> cat sample_data/stocks_data.csv | python bokeh-cli.py --buffer t

        >> python bokeh-cli.py --help
        """
        cli = CLI(
            input_source, output, title, chart_type, series, palette, index, buffer,
            sync_with_source, update_ranges, show_legend, window_size, map_,
            smart_filters, map_zoom, map_layer
        )
        cli.run()
else:
    def cli():
        print("The CLI tool requires click to be installed")

class CLI(object):
    """This is the Bokeh Command Line Interface class and it is in
    charge of providing a very high level access to bokeh charts and
    extends it with functionality.

    """
    def __init__(self, input_source, output, title, chart_type, series, palette,
                 index, buffer, sync_with_source, update_ranges, show_legend,
                 window_size, map_, smart_filters, map_zoom, map_layer):
        """Args:
        input_source (str): path to the series data file (i.e.:
            /source/to/my/data.csv)
            NOTE: this can be either a path to a local file or an url
        output (str, optional): Selects the plotting output, which
            could either be sent to an html file or a bokeh server
            instance. Syntax convention for this option is as follows:
            <output_type>://<type_arg>

            where:
              - output_type: 'file' or 'server'
              - 'file' type options: path_to_output_file
              - 'server' type options syntax: docname[@url][@name]

            Defaults to: --output file://cli_output.html

            Examples:
                --output file://cli_output.html
                --output file:///home/someuser/bokeh_rocks/cli_output.html
                --output server://clidemo

            Default: file://cli_output.html.
        title (str, optional): the title of your chart.
            Default: None.
        chart_type (str, optional): charts classes to use to consume and
            render the input data.
            Default: Line.
        series (str, optional): Name of the series from the input source
            to include in the plot. If not specified all source series
            will be included.
            Defaults to None.
        palette (str, optional): name of the colors palette to use.
            Default: None.
        index (str, optional): Name of the data series to be used as the
            index when plotting. By default the first series found on the
            input file is taken
            Default: None
        buffer (str, optional): if is `t` reads data source as string from
            input buffer using StringIO(sys.stdin.read()) instead of
            reading from a file or an url.
            Default: "f"
        sync_with_source (bool, optional): if True keep the charts source
            created on bokeh-server sync'ed with the source acting like
            `tail -f`.
            Default: False
        window_size (int, optional): show up to N values then start dropping
            off older ones
            Default: '0'

        Attributes:
            source (obj): datasource object for the created chart.
            chart (obj): created chart object.
        """
        self.input = input_source
        self.series = series
        self.index = index
        self.last_byte = -1
        self.sync_with_source = sync_with_source
        self.update_ranges = update_ranges
        self.show_legend = show_legend
        self.window_size = int(window_size)
        self.smart_filters = smart_filters
        self.map_options = {}
        self.current_selection = []

        self.source = self.get_input(input_source, buffer)
        # get the charts specified by the user
        self.factories = create_chart_factories(chart_type)

        if palette:
            print ("Sorry, custom palettes not supported yet, coming soon!")

        # define charts init parameters specified from cmd line and create chart
        self.chart_args = get_chart_params(
            title, output, show_legend=self.show_legend
        )
        if self.smart_filters:
            self.chart_args['tools'] = "pan,wheel_zoom,box_zoom,reset,save," \
                                       "box_select,lasso_select"

        if map_:
            self.map_options['lat'], self.map_options['lng'] = \
                [float(x) for x in map_.strip().split(',')]

            self.map_options['zoom'] = int(map_zoom)
            # Yeah, unfortunate namings.. :-)
            self.map_options['map_type'] = map_layer

    def on_selection_changed(self, obj, attrname, old, new):
        self.current_selection = new

    def limit_source(self, source):
        """ Limit source to cli.window_size, if set.

        Args:
            source (mapping): dict-like object
        """
        if self.window_size:
            for key in source.keys():
                source[key] = source[key][-self.window_size:]

    def run(self):
        """ Start the CLI logic creating the input source, data conversions,
        chart instances to show and all other niceties provided by CLI
        """
        try:
            self.limit_source(self.source)

            children = []
            if self.smart_filters:
                copy_selection = Button(label="copy current selection")
                copy_selection.on_click(self.on_copy)
                children.append(copy_selection)

            self.chart = create_chart(
                self.series, self.source, self.index, self.factories,
                self.map_options, children=children, **self.chart_args
            )
            self.chart.show()

            self.has_ranged_x_axis = 'ranged_x_axis' in self.source.columns
            self.columns = [c for c in self.source.columns if c != 'ranged_x_axis']

            if self.smart_filters:
                for chart in self.chart.charts:
                    chart.source.on_change('selected', self, 'on_selection_changed')
                self.chart.session.poll_document(self.chart.doc)

        except TypeError:
            if not self.series:
                series_list = ', '.join(self.chart.values.keys())
                print(hm.ERR_MSG_TEMPL % series_list)
                raise

        if self.sync_with_source:
            keep_source_input_sync(self.input, self.update_source, self.last_byte)

    def on_copy(self, *args, **kws):
        print("COPYING CONTENT!")
        # TODO: EXPERIMENTAL!!! THIS EXPOSE MANY SECURITY ISSUES AND SHOULD
        #       BE REMOVED ASAP!
        txt = ''
        for rowind in self.current_selection:
            row = self.source.iloc[rowind]
            txt += u"%s\n" % (u",".join(str(row[c]) for c in self.columns))

        os.system("echo '%s' | pbcopy" % txt)

    def update_source(self, new_source):
        """ Update self.chart source with the new data retrieved from
         new_source. It is done by parsing the new source line,
         trasforming it to data to be appended to self.chart source
         updating it on chart.session and actually updating chart.session
         objects.

        Args:
            new_source (str): string that contains the new source row to
                read to the current chart source.
        """
        ns = pd.read_csv(StringIO(new_source), names=self.columns)
        len_source = len(self.source)

        if self.has_ranged_x_axis:
            ns['ranged_x_axis'] = [len_source]
            self.index = 'ranged_x_axis'

        ns.index = [len_source]
        self.source = pd.concat([self.source, ns])

        # TODO: This should be replaced with something that just computes
        #       the new data and source
        fig = create_chart(self.series, ns, self.index, self.factories,
                          self.map_options, **self.chart_args)

        for i, _c in enumerate(fig.charts):
            if not isinstance(_c, bc.GMap):
                # TODO: nested charts are getting ridiculous. Need a better
                #       better interface for charts :-)
                scc = self.chart.charts[i]
                for k, v in _c.source.data.items():
                    scc.source.data[k] = list(scc.source.data[k]) + list(v)

                self.limit_source(scc.source.data)
                chart = scc.chart
                chart.session.store_objects(scc.source)

                if self.update_ranges:
                    plot = chart.plot
                    plot.y_range.start = min(
                        plot.y_range.start, _c.chart.plot.y_range.start
                    )
                    plot.y_range.end = max(
                        plot.y_range.end, _c.chart.plot.y_range.end
                    )
                    plot.x_range.start = min(
                        plot.x_range.start, _c.chart.plot.x_range.start
                    )
                    plot.x_range.end = max(
                        plot.x_range.end, _c.chart.plot.x_range.end
                    )
                    chart.session.store_objects(plot)

    def get_input(self, filepath, buffer):
        """Parse received input options. If buffer is not false (=='f') if
        gets input data from input buffer othewise opens file specified in
        sourcefilename,

        Args:
            filepath (str): path to the file to read from to retrieve data
            buffer (str): if == 't' reads data from input buffer

        Returns:
            string read from filepath/buffer
        """

        if buffer != 'f':
            filepath = StringIO(sys.stdin.read())
        elif filepath is None:
            msg = "No Input! Please specify --source_filename or --buffer t"
            raise IOError(msg)
        else:
            if filepath.lower().startswith('http'):
                # Create a request for the given URL.
                request = urllib2.Request(filepath)
                data = get_data_from_url(request)
                self.last_byte = len(data)

            else:
                filepath = open(filepath, 'r').read()
                self.last_byte = len(filepath)
                filepath = StringIO(filepath)

        source = pd.read_csv(filepath)
        return source

def create_chart(series, source, index, factories, map_options=None, children=None, **args):
    """Create charts instances from types specified in factories using
    data series names, source, index and args

    Args:
        series (list(str)): list of strings specifying the names of the
            series to keep from source
        source (DataFrame): pandas DataFrame with the data series to be
            plotted
        index (str): name of the series of source to be used as index.
        factories (list(ChartObject)): list of chart classes to be used
            to create the charts to be plotted
        **args: arguments to pass to the charts when creating them.
    """
    if not index:
        # if no index was specified as for x axis
        # we take a default "range"
        index = 'ranged_x_axis'
        # add the new x range data to the source dataframe
        source[index] = range(len(source[source.columns[0]]))

    indexes = [x for x in index.split(',') if x]
    data_series = get_data_series(series, source, indexes)
    # parse queries to create the charts..

    charts = []
    for chart_type in factories:
        if chart_type == bc.GMap:
            if not map_options or \
                    not all([x in map_options for x in ['lat', 'lng']]):
                raise ValueError("GMap Charts need lat and lon coordinates!")

            all_args = dict(map_options)
            all_args.update(args)
            chart = chart_type(**all_args)

        else:
            if chart_type == bc.TimeSeries:
                # in case the x axis type is datetime that column must be converted to
                # datetime
                data_series[index] = pd.to_datetime(source[index])

            elif chart_type == bc.Scatter:
                if len(indexes) == 1:
                    scatter_ind = [x for x in data_series.pop(indexes[0]).values]
                    scatter_ind = [scatter_ind] * len(data_series)

                else:
                    scatter_ind = []
                    for key in indexes:
                        scatter_ind.append([x for x in data_series.pop(key).values])

                    if len(scatter_ind) != len(data_series):
                        err_msg = "Number of multiple indexes must be equals" \
                                  " to the number of series"
                        raise ValueError(err_msg)

                for ind, key in enumerate(data_series):
                    values = data_series[key].values
                    data_series[key] = zip(scatter_ind[ind], values)

            chart = chart_type(data_series, **args)
            if hasattr(chart, 'index'):
                chart.index = index

        charts.append(chart)

    fig = bc_utils.Figure(*charts, children=children, **args)
    return fig

def create_chart_factories(chart_types):
    """Receive the chart type(s) specified by the user and build a
    list of the their related functions.

    Args:
        series (str): string that contains the name of the
            chart classes to use when creating the chart, separated by `,`

    example:

    >> create_chart_factories('Line,step')
      [Line, Step]
    """
    return [get_chart(name) for name in chart_types.split(',') if name]

def get_chart(class_name):
    """Return the bokeh class specified in class_name.

    Args:
        class_name (str): name of the chart class to return (i.e.: Line|step)
    """
    return CHARTS_MAP[class_name.strip().lower()]

if __name__ == '__main__':
    cli()


import ROOT
from itertools import izip
from matplotlib import cm
from rootpy.plotting.style.atlas.labels import ATLAS_label
from rootpy.memory.keepalive import keepalive
from .. import ATLAS_LABEL

def set_colors(hists, colors='jet'):
    if isinstance(colors, basestring):
        colors = cm.get_cmap(colors, len(hists))
    if hasattr(colors, '__call__'):
        for i, h in enumerate(hists):
            color = colors((i + 1) / float(len(hists) + 1))
            h.SetColor(color)
    else:
        for h, color in izip(hists, colors):
            h.SetColor(color)

def category_lumi_atlas(pad, category_label=None,
                        data_info=None, atlas_label=None,
                        textsize=20):

    left, right, bottom, top = pad.margin_pixels
    height = float(pad.height_pixels)

    # draw the category label
    if category_label:
        label = ROOT.TLatex(
            1. - pad.GetRightMargin(),
            1. - (textsize - 2) / height,
            category_label)
        label.SetNDC()
        label.SetTextFont(43)
        label.SetTextSize(textsize)
        label.SetTextAlign(31)
        with pad:
            label.Draw()
        keepalive(pad, label)

    # draw the luminosity label
    if data_info is not None:
        plabel = ROOT.TLatex(
            1. - pad.GetLeftMargin() - 0.25,
            1. - (top + textsize + 60) / height,
            str(data_info))
        plabel.SetNDC()
        plabel.SetTextFont(43)
        plabel.SetTextSize(textsize)
        plabel.SetTextAlign(31)
        with pad:
            plabel.Draw()
        keepalive(pad, plabel)

    # draw the ATLAS label
    if atlas_label is not False:
        label = atlas_label or ATLAS_LABEL
        ATLAS_label(pad.GetLeftMargin() + 0.03,
                    1. - (top + textsize + 15) / height,
                    sep=0.132, pad=pad, sqrts=None,
                    text=label,
                    textsize=textsize)
    pad.Update()
    pad.Modified()

def label_plot(pad, template, xaxis, yaxis,
               ylabel='Events', xlabel=None,
               units=None, data_info=None,
               category_label=None,
               atlas_label=None,
               extra_label=None,
               extra_label_position='left',
               textsize=22):

    # set the axis labels
    binw = list(template.xwidth())
    binwidths = list(set(['%.2g' % w for w in binw]))
    if units is not None:
        if xlabel is not None:
            xlabel = '%s [%s]' % (xlabel, units)
        if ylabel and len(binwidths) == 1 and binwidths[0] != '1':
            # constant width bins
            ylabel = '%s / %s %s' % (ylabel, binwidths[0], units)
    elif ylabel and len(binwidths) == 1 and binwidths[0] != '1':
        ylabel = '%s / %s' % (ylabel, binwidths[0])

    if ylabel:
        yaxis.SetTitle(ylabel)
    if xlabel:
        xaxis.SetTitle(xlabel)

    left, right, bottom, top = pad.margin_pixels
    height = float(pad.height_pixels)

    category_lumi_atlas(pad, category_label, data_info, atlas_label)

    # draw the extra label
    if extra_label is not None:
        if extra_label_position == 'left':
            label = ROOT.TLatex(pad.GetLeftMargin() + 0.03,
                                1. - (top + 2 * (textsize + 40)) / height,
                                extra_label)
        else: # right
            label = ROOT.TLatex(1. - pad.GetRightMargin() - 0.03,
                                1. - (top + 2 * (textsize + 40)) / height,
                                extra_label)
            label.SetTextAlign(31)
        label.SetNDC()
        label.SetTextFont(43)
        label.SetTextSize(textsize)
        with pad:
            label.Draw()
        keepalive(pad, label)

    pad.Update()
    pad.Modified()

# class rootpy.plotting.Legend(
#    entries, pad=None,
#    leftmargin=0.5, topmargin=0.05, rightmargin=0.05,
#    entryheight=0.06, entrysep=0.02, margin=0.3,
#    textfont=None, textsize=None, header=None)

def legend_params(position, textsize):
    return dict(
        leftmargin=0.48, topmargin=0.03, rightmargin=0.05,
        entryheight=0.05,
        entrysep=0.01,
        margin=0.25,
        textsize=textsize)


import numpy as np
from numpy.testing import assert_array_almost_equal
from sklearn.neighbors.kd_tree import (KDTree, NeighborsHeap,
                                       simultaneous_sort, kernel_norm,
                                       nodeheap_sort, DTYPE, ITYPE)
from sklearn.neighbors.dist_metrics import DistanceMetric
from sklearn.utils.testing import SkipTest, assert_allclose

V = np.random.random((3, 3))
V = np.dot(V, V.T)

DIMENSION = 3

METRICS = {'euclidean': {},
           'manhattan': {},
           'chebyshev': {},
           'minkowski': dict(p=3)}

def brute_force_neighbors(X, Y, k, metric, **kwargs):
    D = DistanceMetric.get_metric(metric, **kwargs).pairwise(Y, X)
    ind = np.argsort(D, axis=1)[:, :k]
    dist = D[np.arange(Y.shape[0])[:, None], ind]
    return dist, ind

def test_kd_tree_query():
    np.random.seed(0)
    X = np.random.random((40, DIMENSION))
    Y = np.random.random((10, DIMENSION))

    def check_neighbors(dualtree, breadth_first, k, metric, kwargs):
        kdt = KDTree(X, leaf_size=1, metric=metric, **kwargs)
        dist1, ind1 = kdt.query(Y, k, dualtree=dualtree,
                                breadth_first=breadth_first)
        dist2, ind2 = brute_force_neighbors(X, Y, k, metric, **kwargs)

        # don't check indices here: if there are any duplicate distances,
        # the indices may not match.  Distances should not have this problem.
        assert_array_almost_equal(dist1, dist2)

    for (metric, kwargs) in METRICS.items():
        for k in (1, 3, 5):
            for dualtree in (True, False):
                for breadth_first in (True, False):
                    yield (check_neighbors,
                           dualtree, breadth_first,
                           k, metric, kwargs)

def test_kd_tree_query_radius(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    kdt = KDTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind = kdt.query_radius(query_pt, r + eps)[0]
        i = np.where(rad <= r + eps)[0]

        ind.sort()
        i.sort()

        assert_array_almost_equal(i, ind)

def test_kd_tree_query_radius_distance(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    kdt = KDTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind, dist = kdt.query_radius(query_pt, r + eps, return_distance=True)

        ind = ind[0]
        dist = dist[0]

        d = np.sqrt(((query_pt - X[ind]) ** 2).sum(1))

        assert_array_almost_equal(d, dist)

def compute_kernel_slow(Y, X, kernel, h):
    d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
    norm = kernel_norm(h, X.shape[1], kernel)

    if kernel == 'gaussian':
        return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
    elif kernel == 'tophat':
        return norm * (d < h).sum(-1)
    elif kernel == 'epanechnikov':
        return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
    elif kernel == 'exponential':
        return norm * (np.exp(-d / h)).sum(-1)
    elif kernel == 'linear':
        return norm * ((1 - d / h) * (d < h)).sum(-1)
    elif kernel == 'cosine':
        return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
    else:
        raise ValueError('kernel not recognized')

def test_kd_tree_kde(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    kdt = KDTree(X, leaf_size=10)

    for kernel in ['gaussian', 'tophat', 'epanechnikov',
                   'exponential', 'linear', 'cosine']:
        for h in [0.01, 0.1, 1]:
            dens_true = compute_kernel_slow(Y, X, kernel, h)

            def check_results(kernel, h, atol, rtol, breadth_first):
                dens = kdt.kernel_density(Y, h, atol=atol, rtol=rtol,
                                          kernel=kernel,
                                          breadth_first=breadth_first)
                assert_allclose(dens, dens_true, atol=atol,
                                rtol=max(rtol, 1e-7))

            for rtol in [0, 1E-5]:
                for atol in [1E-6, 1E-2]:
                    for breadth_first in (True, False):
                        yield (check_results, kernel, h, atol, rtol,
                               breadth_first)

def test_gaussian_kde(n_samples=1000):
    # Compare gaussian KDE results to scipy.stats.gaussian_kde
    from scipy.stats import gaussian_kde
    np.random.seed(0)
    x_in = np.random.normal(0, 1, n_samples)
    x_out = np.linspace(-5, 5, 30)

    for h in [0.01, 0.1, 1]:
        kdt = KDTree(x_in[:, None])
        try:
            gkde = gaussian_kde(x_in, bw_method=h / np.std(x_in))
        except TypeError:
            raise SkipTest("Old scipy, does not accept explicit bandwidth.")

        dens_kdt = kdt.kernel_density(x_out[:, None], h) / n_samples
        dens_gkde = gkde.evaluate(x_out)

        assert_array_almost_equal(dens_kdt, dens_gkde, decimal=3)

def test_kd_tree_two_point(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    r = np.linspace(0, 1, 10)
    kdt = KDTree(X, leaf_size=10)

    D = DistanceMetric.get_metric("euclidean").pairwise(Y, X)
    counts_true = [(D <= ri).sum() for ri in r]

    def check_two_point(r, dualtree):
        counts = kdt.two_point_correlation(Y, r=r, dualtree=dualtree)
        assert_array_almost_equal(counts, counts_true)

    for dualtree in (True, False):
        yield check_two_point, r, dualtree

def test_kd_tree_pickle():
    import pickle
    np.random.seed(0)
    X = np.random.random((10, 3))
    kdt1 = KDTree(X, leaf_size=1)
    ind1, dist1 = kdt1.query(X)

    def check_pickle_protocol(protocol):
        s = pickle.dumps(kdt1, protocol=protocol)
        kdt2 = pickle.loads(s)
        ind2, dist2 = kdt2.query(X)
        assert_array_almost_equal(ind1, ind2)
        assert_array_almost_equal(dist1, dist2)

    for protocol in (0, 1, 2):
        yield check_pickle_protocol, protocol

def test_neighbors_heap(n_pts=5, n_nbrs=10):
    heap = NeighborsHeap(n_pts, n_nbrs)

    for row in range(n_pts):
        d_in = np.random.random(2 * n_nbrs).astype(DTYPE)
        i_in = np.arange(2 * n_nbrs, dtype=ITYPE)
        for d, i in zip(d_in, i_in):
            heap.push(row, d, i)

        ind = np.argsort(d_in)
        d_in = d_in[ind]
        i_in = i_in[ind]

        d_heap, i_heap = heap.get_arrays(sort=True)

        assert_array_almost_equal(d_in[:n_nbrs], d_heap[row])
        assert_array_almost_equal(i_in[:n_nbrs], i_heap[row])

def test_node_heap(n_nodes=50):
    vals = np.random.random(n_nodes).astype(DTYPE)

    i1 = np.argsort(vals)
    vals2, i2 = nodeheap_sort(vals)

    assert_array_almost_equal(i1, i2)
    assert_array_almost_equal(vals[i1], vals2)

def test_simultaneous_sort(n_rows=10, n_pts=201):
    dist = np.random.random((n_rows, n_pts)).astype(DTYPE)
    ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE)

    dist2 = dist.copy()
    ind2 = ind.copy()

    # simultaneous sort rows using function
    simultaneous_sort(dist, ind)

    # simultaneous sort rows using numpy
    i = np.argsort(dist2, axis=1)
    row_ind = np.arange(n_rows)[:, None]
    dist2 = dist2[row_ind, i]
    ind2 = ind2[row_ind, i]

    assert_array_almost_equal(dist, dist2)
    assert_array_almost_equal(ind, ind2)


# 国债指数：id=0000121;http://pdfm2.eastmoney.com/EM_UBG_PDTI_Fast/api/js?id=0000121&TYPE=k&js=(x)&rtntype=5&isCR=false&fsData1518154947301=fsData1518154947301
# 沪市企业: id=0000131;http://pdfm2.eastmoney.com/EM_UBG_PDTI_Fast/api/js?id=0000131&TYPE=k&js=(x)&rtntype=5&isCR=false&fsData1518156740923=fsData1518156740923
# 深圳企业：id=3994812;http://pdfm2.eastmoney.com/EM_UBG_PDTI_Fast/api/js?id=3994812&TYPE=k&js=(x)&rtntype=5&isCR=false&fsData1518156947700=fsData1518156947700

import urllib.request
import pandas as pd
import json

class Bond(object):
	"""docstring for Bond"""
	def __init__(self):
		self.index = {
			'国债指数':'0000121',
			'沪市企业债':'0000131',
			'深圳企业债':'3994812'
		}

	def bond_index(self,id):
		url = r'http://pdfm2.eastmoney.com/EM_UBG_PDTI_Fast/api/js?id={}&TYPE=k&js=(x)&rtntype=5&isCR=false&fsData1518154947301=fsData1518154947301'.format(id)
		raw = json.loads(urllib.request.urlopen(url).read())
		head = ['日期','开盘','收盘','最高','最低','成交量','成交金额','振幅']
		return pd.DataFrame(list(map(lambda x:x.split(','),raw['data'])),columns=head)


# -*- coding: utf-8 -*-
"""
Nearest Centroid Classification
"""

# Author: Robert Layton <robertlayton@gmail.com>
#         Olivier Grisel <olivier.grisel@ensta.org>
#
# License: BSD 3 clause

import warnings
import numpy as np
from scipy import sparse as sp

from ..base import BaseEstimator, ClassifierMixin
from ..metrics.pairwise import pairwise_distances
from ..preprocessing import LabelEncoder
from ..utils.validation import check_array, check_X_y, check_is_fitted
from ..utils.sparsefuncs import csc_median_axis_0
from ..utils.multiclass import check_classification_targets

class NearestCentroid(BaseEstimator, ClassifierMixin):
    """Nearest centroid classifier.

    Each class is represented by its centroid, with test samples classified to
    the class with the nearest centroid.

    Read more in the :ref:`User Guide <nearest_centroid_classifier>`.

    Parameters
    ----------
    metric: string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.pairwise_distances for its
        metric parameter.
        The centroids for the samples corresponding to each class is the point
        from which the sum of the distances (according to the metric) of all
        samples that belong to that particular class are minimized.
        If the "manhattan" metric is provided, this centroid is the median and
        for all other metrics, the centroid is now set to be the mean.

    shrink_threshold : float, optional (default = None)
        Threshold for shrinking centroids to remove features.

    Attributes
    ----------
    centroids_ : array-like, shape = [n_classes, n_features]
        Centroid of each class

    Examples
    --------
    >>> from sklearn.neighbors.nearest_centroid import NearestCentroid
    >>> import numpy as np
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> y = np.array([1, 1, 1, 2, 2, 2])
    >>> clf = NearestCentroid()
    >>> clf.fit(X, y)
    NearestCentroid(metric='euclidean', shrink_threshold=None)
    >>> print(clf.predict([[-0.8, -1]]))
    [1]

    See also
    --------
    sklearn.neighbors.KNeighborsClassifier: nearest neighbors classifier

    Notes
    -----
    When used for text classification with tf-idf vectors, this classifier is
    also known as the Rocchio classifier.

    References
    ----------
    Tibshirani, R., Hastie, T., Narasimhan, B., & Chu, G. (2002). Diagnosis of
    multiple cancer types by shrunken centroids of gene expression. Proceedings
    of the National Academy of Sciences of the United States of America,
    99(10), 6567-6572. The National Academy of Sciences.

    """

    def __init__(self, metric='euclidean', shrink_threshold=None):
        self.metric = metric
        self.shrink_threshold = shrink_threshold

    def fit(self, X, y):
        """
        Fit the NearestCentroid model according to the given training data.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.
            Note that centroid shrinking cannot be used with sparse matrices.
        y : array, shape = [n_samples]
            Target values (integers)
        """
        # If X is sparse and the metric is "manhattan", store it in a csc
        # format is easier to calculate the median.
        if self.metric == 'manhattan':
            X, y = check_X_y(X, y, ['csc'])
        else:
            X, y = check_X_y(X, y, ['csr', 'csc'])
        is_X_sparse = sp.issparse(X)
        if is_X_sparse and self.shrink_threshold:
            raise ValueError("threshold shrinking not supported"
                             " for sparse input")
        check_classification_targets(y)

        n_samples, n_features = X.shape
        le = LabelEncoder()
        y_ind = le.fit_transform(y)
        self.classes_ = classes = le.classes_
        n_classes = classes.size
        if n_classes < 2:
            raise ValueError('y has less than 2 classes')

        # Mask mapping each class to it's members.
        self.centroids_ = np.empty((n_classes, n_features), dtype=np.float64)
        # Number of clusters in each class.
        nk = np.zeros(n_classes)

        for cur_class in range(n_classes):
            center_mask = y_ind == cur_class
            nk[cur_class] = np.sum(center_mask)
            if is_X_sparse:
                center_mask = np.where(center_mask)[0]

            # XXX: Update other averaging methods according to the metrics.
            if self.metric == "manhattan":
                # NumPy does not calculate median of sparse matrices.
                if not is_X_sparse:
                    self.centroids_[cur_class] = np.median(X[center_mask], axis=0)
                else:
                    self.centroids_[cur_class] = csc_median_axis_0(X[center_mask])
            else:
                if self.metric != 'euclidean':
                    warnings.warn("Averaging for metrics other than "
                                  "euclidean and manhattan not supported. "
                                  "The average is set to be the mean."
                                  )
                self.centroids_[cur_class] = X[center_mask].mean(axis=0)

        if self.shrink_threshold:
            dataset_centroid_ = np.mean(X, axis=0)

            # m parameter for determining deviation
            m = np.sqrt((1. / nk) + (1. / n_samples))
            # Calculate deviation using the standard deviation of centroids.
            variance = (X - self.centroids_[y_ind]) ** 2
            variance = variance.sum(axis=0)
            s = np.sqrt(variance / (n_samples - n_classes))
            s += np.median(s)  # To deter outliers from affecting the results.
            mm = m.reshape(len(m), 1)  # Reshape to allow broadcasting.
            ms = mm * s
            deviation = ((self.centroids_ - dataset_centroid_) / ms)
            # Soft thresholding: if the deviation crosses 0 during shrinking,
            # it becomes zero.
            signs = np.sign(deviation)
            deviation = (np.abs(deviation) - self.shrink_threshold)
            deviation[deviation < 0] = 0
            deviation *= signs
            # Now adjust the centroids using the deviation
            msd = ms * deviation
            self.centroids_ = dataset_centroid_[np.newaxis, :] + msd
        return self

    def predict(self, X):
        """Perform classification on an array of test vectors X.

        The predicted class C for each sample in X is returned.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = [n_samples]

        Notes
        -----
        If the metric constructor parameter is "precomputed", X is assumed to
        be the distance matrix between the data to be predicted and
        ``self.centroids_``.
        """
        check_is_fitted(self, 'centroids_')

        X = check_array(X, accept_sparse='csr')
        return self.classes_[pairwise_distances(
            X, self.centroids_, metric=self.metric).argmin(axis=1)]