"""
A collection of modules for collecting, analyzing and plotting
financial data.   User contributions welcome!

"""
#from __future__ import division
import os, time, warnings
from urllib import urlopen

try:
    from hashlib import md5
except ImportError:
    from md5 import md5 #Deprecated in 2.5

try: import datetime
except ImportError:
    raise ImportError('The finance module requires datetime support (python2.3)')

import numpy as np

from matplotlib import verbose, get_configdir
from dates import date2num
from matplotlib.cbook import Bunch
from matplotlib.collections import LineCollection, PolyCollection
from matplotlib.colors import colorConverter
from lines import Line2D, TICKLEFT, TICKRIGHT
from patches import Rectangle
from matplotlib.transforms import Affine2D

configdir = get_configdir()
cachedir = os.path.join(configdir, 'finance.cache')

def parse_yahoo_historical(fh, asobject=False, adjusted=True):
    """
    Parse the historical data in file handle fh from yahoo finance and return
    results as a list of

    d, open, close, high, low, volume

    where d is a floating poing representation of date, as returned by date2num

    if adjust=True, use adjusted prices
    """
    results = []

    lines = fh.readlines()

    datefmt = None

    for line in lines[1:]:

        vals = line.split(',')

        if len(vals)!=7: continue
        datestr = vals[0]
        if datefmt is None:
            try:
                datefmt = '%Y-%m-%d'
                dt = datetime.date(*time.strptime(datestr, datefmt)[:3])
            except ValueError:
                datefmt = '%d-%b-%y'  # Old Yahoo--cached file?
        dt = datetime.date(*time.strptime(datestr, datefmt)[:3])
        d = date2num(dt)
        open, high, low, close =  [float(val) for val in vals[1:5]]
        volume = int(vals[5])
        if adjusted:
            aclose = float(vals[6])
            m = aclose/close
            open *= m
            high *= m
            low *= m
            close = aclose

        results.append((d, open, close, high, low, volume))
    results.reverse()
    if asobject:
        if len(results)==0: return None
        else:
            date, open, close, high, low, volume = map(np.asarray, zip(*results))
        return Bunch(date=date, open=open, close=close, high=high, low=low, volume=volume)
    else:

        return results

def fetch_historical_yahoo(ticker, date1, date2, cachename=None):
    """
    Fetch historical data for ticker between date1 and date2.  date1 and
    date2 are datetime instances

    Ex:
    fh = fetch_historical_yahoo('^GSPC', d1, d2)

    cachename is the name of the local file cache.  If None, will
    default to the md5 hash or the url (which incorporates the ticker
    and date range)

    a file handle is returned
    """

    ticker = ticker.upper()

    d1 = (date1.month-1, date1.day, date1.year)
    d2 = (date2.month-1, date2.day, date2.year)

    urlFmt = 'http://table.finance.yahoo.com/table.csv?a=%d&b=%d&c=%d&d=%d&e=%d&f=%d&s=%s&y=0&g=d&ignore=.csv'

    url =  urlFmt % (d1[0], d1[1], d1[2],
                     d2[0], d2[1], d2[2], ticker)

    if cachename is None:
        cachename = os.path.join(cachedir, md5(url).hexdigest())
    if os.path.exists(cachename):
        fh = file(cachename)
        verbose.report('Using cachefile %s for %s'%(cachename, ticker))
    else:
        if not os.path.isdir(cachedir): os.mkdir(cachedir)
        fh = file(cachename, 'w')
        fh.write(urlopen(url).read())
        fh.close()
        verbose.report('Saved %s data to cache file %s'%(ticker, cachename))
        fh = file(cachename, 'r')

    return fh

def quotes_historical_yahoo(ticker, date1, date2, asobject=False, adjusted=True, cachename=None):
    """
    Get historical data for ticker between date1 and date2.  date1 and
    date2 are datetime instances

    results are a list of tuples

      (d, open, close, high, low, volume)

    where d is a floating poing representation of date, as returned by date2num

    if asobject is True, the return val is an object with attrs date,
    open, close, high, low, volume, which are equal length arrays

    if adjust=True, use adjusted prices

    Ex:
    sp = f.quotes_historical_yahoo('^GSPC', d1, d2, asobject=True, adjusted=True)
    returns = (sp.open[1:] - sp.open[:-1])/sp.open[1:]
    [n,bins,patches] = hist(returns, 100)
    mu = mean(returns)
    sigma = std(returns)
    x = normpdf(bins, mu, sigma)
    plot(bins, x, color='red', lw=2)

    cachename is the name of the local file cache.  If None, will
    default to the md5 hash or the url (which incorporates the ticker
    and date range)
    """

    fh = fetch_historical_yahoo(ticker, date1, date2, cachename)

    try: ret = parse_yahoo_historical(fh, asobject, adjusted)
    except IOError, exc:
        warnings.warn('urlopen() failure\n' + url + '\n' + exc.strerror[1])
        return None

    return ret

def plot_day_summary(ax, quotes, ticksize=3,
                     colorup='k', colordown='r',
                     ):
    """
    quotes is a list of (time, open, close, high, low, ...) tuples

    Represent the time, open, close, high, low as a vertical line
    ranging from low to high.  The left tick is the open and the right
    tick is the close.

    time must be in float date format - see date2num

    ax          : an Axes instance to plot to
    ticksize    : open/close tick marker in points
    colorup     : the color of the lines where close >= open
    colordown   : the color of the lines where close <  open
    return value is a list of lines added
    """

    lines = []
    for q in quotes:

        t, open, close, high, low = q[:5]

        if close>=open : color = colorup
        else           : color = colordown

        vline = Line2D(
            xdata=(t, t), ydata=(low, high),
            color=color,
            antialiased=False,   # no need to antialias vert lines
            )

        oline = Line2D(
            xdata=(t, t), ydata=(open, open),
            color=color,
            antialiased=False,
            marker=TICKLEFT,
            markersize=ticksize,
            )

        cline = Line2D(
            xdata=(t, t), ydata=(close, close),
            color=color,
            antialiased=False,
            markersize=ticksize,
            marker=TICKRIGHT)

        lines.extend((vline, oline, cline))
        ax.add_line(vline)
        ax.add_line(oline)
        ax.add_line(cline)

    ax.autoscale_view()

    return lines

def candlestick(ax, quotes, width=0.2, colorup='k', colordown='r',
                alpha=1.0):

    """

    quotes is a list of (time, open, close, high, low, ...)  tuples.
    As long as the first 5 elements of the tuples are these values,
    the tuple can be as long as you want (eg it may store volume).

    time must be in float days format - see date2num

    Plot the time, open, close, high, low as a vertical line ranging
    from low to high.  Use a rectangular bar to represent the
    open-close span.  If close >= open, use colorup to color the bar,
    otherwise use colordown

    ax          : an Axes instance to plot to
    width       : fraction of a day for the rectangle width
    colorup     : the color of the rectangle where close >= open
    colordown   : the color of the rectangle where close <  open
    alpha       : the rectangle alpha level

    return value is lines, patches where lines is a list of lines
    added and patches is a list of the rectangle patches added
    """

    OFFSET = width/2.0

    lines = []
    patches = []
    for q in quotes:
        t, open, close, high, low = q[:5]

        if close>=open :
            color = colorup
            lower = open
            height = close-open
        else           :
            color = colordown
            lower = close
            height = open-close

        vline = Line2D(
            xdata=(t, t), ydata=(low, high),
            color='k',
            linewidth=0.5,
            antialiased=True,
            )

        rect = Rectangle(
            xy    = (t-OFFSET, lower),
            width = width,
            height = height,
            facecolor = color,
            edgecolor = color,
            )
        rect.set_alpha(alpha)

        lines.append(vline)
        patches.append(rect)
        ax.add_line(vline)
        ax.add_patch(rect)
    ax.autoscale_view()

    return lines, patches

def plot_day_summary2(ax, opens, closes, highs, lows, ticksize=4,
                      colorup='k', colordown='r',
                     ):
    """

    Represent the time, open, close, high, low as a vertical line
    ranging from low to high.  The left tick is the open and the right
    tick is the close.

    ax          : an Axes instance to plot to
    ticksize    : size of open and close ticks in points
    colorup     : the color of the lines where close >= open
    colordown   : the color of the lines where close <  open

    return value is a list of lines added
    """

    # note this code assumes if any value open, close, low, high is
    # missing they all are missing

    rangeSegments = [ ((i, low), (i, high)) for i, low, high in zip(xrange(len(lows)), lows, highs) if low != -1 ]

    # the ticks will be from ticksize to 0 in points at the origin and
    # we'll translate these to the i, close location
    openSegments = [  ((-ticksize, 0), (0, 0)) ]

    # the ticks will be from 0 to ticksize in points at the origin and
    # we'll translate these to the i, close location
    closeSegments = [ ((0, 0), (ticksize, 0)) ]

    offsetsOpen = [ (i, open) for i, open in zip(xrange(len(opens)), opens) if open != -1 ]

    offsetsClose = [ (i, close) for i, close in zip(xrange(len(closes)), closes) if close != -1 ]

    scale = ax.figure.dpi * (1.0/72.0)

    tickTransform = Affine2D().scale(scale, 0.0)

    r,g,b = colorConverter.to_rgb(colorup)
    colorup = r,g,b,1
    r,g,b = colorConverter.to_rgb(colordown)
    colordown = r,g,b,1
    colord = { True : colorup,
               False : colordown,
               }
    colors = [colord[open<close] for open, close in zip(opens, closes) if open!=-1 and close !=-1]

    assert(len(rangeSegments)==len(offsetsOpen))
    assert(len(offsetsOpen)==len(offsetsClose))
    assert(len(offsetsClose)==len(colors))

    useAA = 0,   # use tuple here
    lw = 1,      # and here
    rangeCollection = LineCollection(rangeSegments,
                                     colors       = colors,
                                     linewidths   = lw,
                                     antialiaseds = useAA,
                                     )

    openCollection = LineCollection(openSegments,
                                    colors       = colors,
                                    antialiaseds = useAA,
                                    linewidths   = lw,
                                    offsets      = offsetsOpen,
                                    transOffset  = ax.transData,
                                   )
    openCollection.set_transform(tickTransform)

    closeCollection = LineCollection(closeSegments,
                                     colors       = colors,
                                     antialiaseds = useAA,
                                     linewidths   = lw,
                                     offsets      = offsetsClose,
                                     transOffset  = ax.transData,
                                     )
    closeCollection.set_transform(tickTransform)

    minpy, maxx = (0, len(rangeSegments))
    miny = min([low for low in lows if low !=-1])
    maxy = max([high for high in highs if high != -1])
    corners = (minpy, miny), (maxx, maxy)
    ax.update_datalim(corners)
    ax.autoscale_view()

    # add these last
    ax.add_collection(rangeCollection)
    ax.add_collection(openCollection)
    ax.add_collection(closeCollection)
    return rangeCollection, openCollection, closeCollection

def candlestick2(ax, opens, closes, highs, lows, width=4,
                 colorup='k', colordown='r',
                 alpha=0.75,
                ):
    """

    Represent the open, close as a bar line and high low range as a
    vertical line.

    ax          : an Axes instance to plot to
    width       : the bar width in points
    colorup     : the color of the lines where close >= open
    colordown   : the color of the lines where close <  open
    alpha       : bar transparency

    return value is lineCollection, barCollection
    """

    # note this code assumes if any value open, close, low, high is
    # missing they all are missing

    delta = width/2.
    barVerts = [ ( (i-delta, open), (i-delta, close), (i+delta, close), (i+delta, open) ) for i, open, close in zip(xrange(len(opens)), opens, closes) if open != -1 and close!=-1 ]

    rangeSegments = [ ((i, low), (i, high)) for i, low, high in zip(xrange(len(lows)), lows, highs) if low != -1 ]

    r,g,b = colorConverter.to_rgb(colorup)
    colorup = r,g,b,alpha
    r,g,b = colorConverter.to_rgb(colordown)
    colordown = r,g,b,alpha
    colord = { True : colorup,
               False : colordown,
               }
    colors = [colord[open<close] for open, close in zip(opens, closes) if open!=-1 and close !=-1]

    assert(len(barVerts)==len(rangeSegments))

    useAA = 0,  # use tuple here
    lw = 0.5,   # and here
    rangeCollection = LineCollection(rangeSegments,
                                     colors       = ( (0,0,0,1), ),
                                     linewidths   = lw,
                                     antialiaseds = useAA,
                                     )

    barCollection = PolyCollection(barVerts,
                                   facecolors   = colors,
                                   edgecolors   = ( (0,0,0,1), ),
                                   antialiaseds = useAA,
                                   linewidths   = lw,
                                   )

    minx, maxx = 0, len(rangeSegments)
    miny = min([low for low in lows if low !=-1])
    maxy = max([high for high in highs if high != -1])

    corners = (minx, miny), (maxx, maxy)
    ax.update_datalim(corners)
    ax.autoscale_view()

    # add these last
    ax.add_collection(barCollection)
    ax.add_collection(rangeCollection)
    return rangeCollection, barCollection

def volume_overlay(ax, opens, closes, volumes,
                   colorup='k', colordown='r',
                   width=4, alpha=1.0):
    """
    Add a volume overlay to the current axes.  The opens and closes
    are used to determine the color of the bar.  -1 is missing.  If a
    value is missing on one it must be missing on all

    ax          : an Axes instance to plot to
    width       : the bar width in points
    colorup     : the color of the lines where close >= open
    colordown   : the color of the lines where close <  open
    alpha       : bar transparency

    """

    r,g,b = colorConverter.to_rgb(colorup)
    colorup = r,g,b,alpha
    r,g,b = colorConverter.to_rgb(colordown)
    colordown = r,g,b,alpha
    colord = { True : colorup,
               False : colordown,
               }
    colors = [colord[open<close] for open, close in zip(opens, closes) if open!=-1 and close !=-1]

    delta = width/2.
    bars = [ ( (i-delta, 0), (i-delta, v), (i+delta, v), (i+delta, 0)) for i, v in enumerate(volumes) if v != -1 ]

    barCollection = PolyCollection(bars,
                                   facecolors   = colors,
                                   edgecolors   = ( (0,0,0,1), ),
                                   antialiaseds = (0,),
                                   linewidths   = (0.5,),
                                   )

    corners = (0, 0), (len(bars), max(volumes))
    ax.update_datalim(corners)
    ax.autoscale_view()

    # add these last
    return barCollection

def volume_overlay2(ax, closes, volumes,
                   colorup='k', colordown='r',
                   width=4, alpha=1.0):
    """
    Add a volume overlay to the current axes.  The closes are used to
    determine the color of the bar.  -1 is missing.  If a value is
    missing on one it must be missing on all

    ax          : an Axes instance to plot to
    width       : the bar width in points
    colorup     : the color of the lines where close >= open
    colordown   : the color of the lines where close <  open
    alpha       : bar transparency

    nb: first point is not displayed - it is used only for choosing the
    right color

    """

    return volume_overlay(ax,closes[:-1],closes[1:],volumes[1:],colorup,colordown,width,alpha)

def volume_overlay3(ax, quotes,
                   colorup='k', colordown='r',
                   width=4, alpha=1.0):
    """
    Add a volume overlay to the current axes.  quotes is a list of (d,
    open, close, high, low, volume) and close-open is used to
    determine the color of the bar

    kwarg
    width       : the bar width in points
    colorup     : the color of the lines where close1 >= close0
    colordown   : the color of the lines where close1 <  close0
    alpha       : bar transparency

    """

    r,g,b = colorConverter.to_rgb(colorup)
    colorup = r,g,b,alpha
    r,g,b = colorConverter.to_rgb(colordown)
    colordown = r,g,b,alpha
    colord = { True : colorup,
               False : colordown,
               }

    dates, opens, closes, highs, lows, volumes = zip(*quotes)
    colors = [colord[close1>=close0] for close0, close1 in zip(closes[:-1], closes[1:]) if close0!=-1 and close1 !=-1]
    colors.insert(0,colord[closes[0]>=opens[0]])

    right = width/2.0
    left = -width/2.0

    bars = [ ( (left, 0), (left, volume), (right, volume), (right, 0)) for d, open, close, high, low, volume in quotes]

    sx = ax.figure.dpi * (1.0/72.0)  # scale for points
    sy = ax.bbox.height / ax.viewLim.height

    barTransform = Affine2D().scale(sx,sy)

    dates = [d for d, open, close, high, low, volume in quotes]
    offsetsBars = [(d, 0) for d in dates]

    useAA = 0,  # use tuple here
    lw = 0.5,   # and here
    barCollection = PolyCollection(bars,
                                   facecolors   = colors,
                                   edgecolors   = ( (0,0,0,1), ),
                                   antialiaseds = useAA,
                                   linewidths   = lw,
                                   offsets      = offsetsBars,
                                   transOffset  = ax.transData,
                                   )
    barCollection.set_transform(barTransform)

    minpy, maxx = (min(dates), max(dates))
    miny = 0
    maxy = max([volume for d, open, close, high, low, volume in quotes])
    corners = (minpy, miny), (maxx, maxy)
    ax.update_datalim(corners)
    #print 'datalim', ax.dataLim.get_bounds()
    #print 'viewlim', ax.viewLim.get_bounds()

    ax.add_collection(barCollection)
    ax.autoscale_view()

    return barCollection

def index_bar(ax, vals,
              facecolor='b', edgecolor='l',
              width=4, alpha=1.0, ):
    """
    Add a bar collection graph with height vals (-1 is missing).

    ax          : an Axes instance to plot to
    width       : the bar width in points
    alpha       : bar transparency

    """

    facecolors = (colorConverter.to_rgba(facecolor, alpha),)
    edgecolors = (colorConverter.to_rgba(edgecolor, alpha),)

    right = width/2.0
    left = -width/2.0

    bars = [ ( (left, 0), (left, v), (right, v), (right, 0)) for v in vals if v != -1 ]

    sx = ax.figure.dpi * (1.0/72.0)  # scale for points
    sy = ax.bbox.height / ax.viewLim.height

    barTransform = Affine2D().scale(sx,sy)

    offsetsBars = [ (i, 0) for i,v in enumerate(vals) if v != -1 ]

    barCollection = PolyCollection(bars,
                                   facecolors   = facecolors,
                                   edgecolors   = edgecolors,
                                   antialiaseds = (0,),
                                   linewidths   = (0.5,),
                                   offsets      = offsetsBars,
                                   transOffset  = ax.transData,
                                   )
    barCollection.set_transform(barTransform)

    minpy, maxx = (0, len(offsetsBars))
    miny = 0
    maxy = max([v for v in vals if v!=-1])
    corners = (minpy, miny), (maxx, maxy)
    ax.update_datalim(corners)
    ax.autoscale_view()

    # add these last
    ax.add_collection(barCollection)
    return barCollection


"""
========================
Partial Dependence Plots
========================

Partial dependence plots show the dependence between the target function [1]_
and a set of 'target' features, marginalizing over the
values of all other features (the complement features). Due to the limits
of human perception the size of the target feature set must be small (usually,
one or two) thus the target features are usually chosen among the most
important features
(see :attr:`~sklearn.ensemble.GradientBoostingRegressor.feature_importances_`).

This example shows how to obtain partial dependence plots from a
:class:`~sklearn.ensemble.GradientBoostingRegressor` trained on the California
housing dataset. The example is taken from [HTF2009]_.

The plot shows four one-way and one two-way partial dependence plots.
The target variables for the one-way PDP are:
median income (`MedInc`), avg. occupants per household (`AvgOccup`),
median house age (`HouseAge`), and avg. rooms per household (`AveRooms`).

We can clearly see that the median house price shows a linear relationship
with the median income (top left) and that the house price drops when the
avg. occupants per household increases (top middle).
The top right plot shows that the house age in a district does not have
a strong influence on the (median) house price; so does the average rooms
per household.
The tick marks on the x-axis represent the deciles of the feature values
in the training data.

Partial dependence plots with two target features enable us to visualize
interactions among them. The two-way partial dependence plot shows the
dependence of median house price on joint values of house age and avg.
occupants per household. We can clearly see an interaction between the
two features:
For an avg. occupancy greater than two, the house price is nearly independent
of the house age, whereas for values less than two there is a strong dependence
on age.

.. [HTF2009] T. Hastie, R. Tibshirani and J. Friedman,
    "Elements of Statistical Learning Ed. 2", Springer, 2009.

.. [1] For classification you can think of it as the regression score before
       the link function.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from mpl_toolkits.mplot3d import Axes3D

from sklearn.cross_validation import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble.partial_dependence import plot_partial_dependence
from sklearn.ensemble.partial_dependence import partial_dependence
from sklearn.datasets.california_housing import fetch_california_housing

# fetch California housing dataset
cal_housing = fetch_california_housing()

# split 80/20 train-test
X_train, X_test, y_train, y_test = train_test_split(cal_housing.data,
                                                    cal_housing.target,
                                                    test_size=0.2,
                                                    random_state=1)
names = cal_housing.feature_names

print('_' * 80)
print("Training GBRT...")
clf = GradientBoostingRegressor(n_estimators=100, max_depth=4,
                                learning_rate=0.1, loss='huber',
                                random_state=1)
clf.fit(X_train, y_train)
print("done.")

print('_' * 80)
print('Convenience plot with ``partial_dependence_plots``')
print

features = [0, 5, 1, 2, (5, 1)]
fig, axs = plot_partial_dependence(clf, X_train, features, feature_names=names,
                                   n_jobs=3, grid_resolution=50)
fig.suptitle('Partial dependence of house value on nonlocation features\n'
             'for the California housing dataset')
plt.subplots_adjust(top=0.9)  # tight_layout causes overlap with suptitle

print('_' * 80)
print('Custom 3d plot via ``partial_dependence``')
print
fig = plt.figure()

target_feature = (1, 5)
pdp, (x_axis, y_axis) = partial_dependence(clf, target_feature,
                                           X=X_train, grid_resolution=50)
XX, YY = np.meshgrid(x_axis, y_axis)
Z = pdp.T.reshape(XX.shape).T
ax = Axes3D(fig)
surf = ax.plot_surface(XX, YY, Z, rstride=1, cstride=1, cmap=plt.cm.BuPu)
ax.set_xlabel(names[target_feature[0]])
ax.set_ylabel(names[target_feature[1]])
ax.set_zlabel('Partial dependence')
#  pretty init view
ax.view_init(elev=22, azim=122)
plt.colorbar(surf)
plt.suptitle('Partial dependence of house value on median age and '
            'average occupancy')
plt.subplots_adjust(top=0.9)

plt.show()


import pandas as pd

from bokeh.io import output_file, show
from bokeh.models import BasicTicker, ColorBar, ColumnDataSource, LinearColorMapper, PrintfTickFormatter
from bokeh.plotting import figure
from bokeh.sampledata.unemployment1948 import data
from bokeh.transform import transform

output_file("unemploymemt.html")

data.Year = data.Year.astype(str)
data = data.set_index('Year')
data.drop('Annual', axis=1, inplace=True)
data.columns.name = 'Month'

# reshape to 1D array or rates with a month and year for each row.
df = pd.DataFrame(data.stack(), columns=['rate']).reset_index()

source = ColumnDataSource(df)

# this is the colormap from the original NYTimes plot
colors = ["#75968f", "#a5bab7", "#c9d9d3", "#e2e2e2", "#dfccce", "#ddb7b1", "#cc7878", "#933b41", "#550b1d"]
mapper = LinearColorMapper(palette=colors, low=df.rate.min(), high=df.rate.max())

p = figure(plot_width=800, plot_height=300, title="US Unemployment 1948—2016",
           x_range=list(data.index), y_range=list(reversed(data.columns)),
           toolbar_location=None, tools="", x_axis_location="above")

p.rect(x="Year", y="Month", width=1, height=1, source=source,
       line_color=None, fill_color=transform('rate', mapper))

color_bar = ColorBar(color_mapper=mapper, location=(0, 0),
                     ticker=BasicTicker(desired_num_ticks=len(colors)),
                     formatter=PrintfTickFormatter(format="%d%%"))

p.add_layout(color_bar, 'right')

p.axis.axis_line_color = None
p.axis.major_tick_line_color = None
p.axis.major_label_text_font_size = "5pt"
p.axis.major_label_standoff = 0
p.xaxis.major_label_orientation = 1.0

show(p)


# -*- coding: utf-8 -*-
import os
import string
import json
import pandas as pd
import numpy as np
from scipy.stats.stats import pearsonr
from dateutil.parser import parse
from common import globals as glob

SPECIAL_CHARS = string.punctuation.replace("_", "")
SPECIAL_CHARS = string.punctuation.replace("-", "")
SPECIAL_CHARS = set(SPECIAL_CHARS)

def get_quality_summary(qual):
    summary = {}
    #we are only intrested in the dqs (data quality score) under each category
    for q in qual.keys():
        summary[q] = {}
        summary[q]['dqs'] = qual[q]['dqs']
    return summary

def contains_special_chars(word):
    #glob.log.info(word)
    if any(char in SPECIAL_CHARS for char in word):
        return True
    else:
        return False

def encode_str_in_csv(line, s):
    line += '\"' + s + '\"' + ','
    return line

def store_in_dict(d, k, key1, val1, key2, val2):
    d[k] = {}
    d[k][key1] = val1
    d[k][key2] = val2

def check_date(qual, category, df, field):
    glob.log.info('checking %s ...' %(field))
    #just a simple check to validate date
    num_invalid = 0
    for i in range(len(df)):
        store = df.ix[i]
        try:
            parse(store[field])
        except Exception as e:
            num_invalid += 1
            glob.log.error('found an invalid %s -> %s for store id %s' % (field, str(store[field]), str(store['store_id'])))
            glob.log.error(str(e))

    qual[category][field] = {}
    qual[category][field]['count']   = num_invalid
    qual[category][field]['percent'] = round(100*(float(num_invalid)/len(df)), glob.PRECISION)
    return num_invalid

def check_as_string_wo_special_chars(qual, category, df, field, prim_key_field):
    glob.log.info('checking %s ...' %(field))
    #just a simple check to say should not contain any special characters
    num_invalid = 0
    for i in range(len(df)):
        store = df.ix[i]
        if contains_special_chars(str(store[field])) == True:
            num_invalid += 1
            glob.log.error('found an invalid %s -> %s for store id %s' % (field, str(store[field]), str(store[prim_key_field])))

    qual[category][field] = {}
    qual[category][field]['count']   = num_invalid
    qual[category][field]['percent'] = round(100*(float(num_invalid)/len(df)), glob.PRECISION)
    return num_invalid

def check_as_numeric(qual, category, df, field):
    glob.log.info('checking %s ...' %(field))
    #just a simple check to say field should be numeric
    num_invalid = 0
    for i in range(len(df)):
        row = df.ix[i]
        val = str(row[field])
        try:
            float(val)
        except Exception as e:
            num_invalid += 1
            glob.log.error('found an invalid %s -> %s' % (field, str(row[field])))

    qual[category][field] = {}
    qual[category][field]['count']   = num_invalid
    qual[category][field]['percent'] = round(100*(float(num_invalid)/len(df)), glob.PRECISION)
    return num_invalid

def check_missing(qual, df, mf = []):
    glob.log.info('checking missing data...')
    qual['missing_data'] = {}

    #check missing fraction of missing rows in each column and also overall
    total_cell_count = df.shape[0] * df.shape[1] #rows x columns
    total_empty_cells = 0
    mandatory_cells_empty_count = 0 #count of empty cells in mandatory columns
    mandatory_feature_list_provided = (True if len(mf) != 0 else False)

    for col in df.columns:
        #find out the number of columns that are empty, the idea is that
        #we would end up with a TRUE/FALSE array and then summing it up
        #gives the count of FALSE or empty cells
        empty_cells        = df[col].isnull()
        num_empty_cells    = sum(empty_cells)
        total_empty_cells += num_empty_cells
        total_cells        = len(empty_cells)

        #if mandatory feature list provided then check if this feature is mandatory
        #if no specific list is provided then consider all features as mandatory
        if mandatory_feature_list_provided == True:
            if col in mf:
                mandatory_cells_empty_count += num_empty_cells
        else:
            mandatory_cells_empty_count += num_empty_cells

        fraction_empty  = 100*(float(num_empty_cells)/(total_cells))
        #store this info in the dict if there were any empty cells
        if num_empty_cells != 0:
            store_in_dict(qual['missing_data'], col, 'percent',
                          round(fraction_empty, glob.PRECISION),
                          'count', num_empty_cells)

    #overall empty cell fraction
    fraction_empty = 100*(float(total_empty_cells)/(total_cell_count))
    store_in_dict(qual['missing_data'], 'overall', 'percent',
                  round(fraction_empty, glob.PRECISION),
                  'count', total_empty_cells)

    #calculate two data quality scores; a raw score calculated simply as
    # 1 - (total missing values/total values) and an adjusted score which
    #is calculated based on missing values that really matter i.e. which would
    #cause the entire row to get discarded.
    raw_score = round(100 - fraction_empty, glob.PRECISION)
    if mandatory_feature_list_provided == True:
        adjusted_raw_score = round(100 - ((float(mandatory_cells_empty_count)/total_cell_count)))
    else:
        #in case no mandatory features were provided then adjusted score is same as raw score
        adjusted_raw_score = raw_score

    qual['missing_data']['dqs'] = {}
    qual['missing_data']['dqs']['raw_score']      = raw_score
    qual['missing_data']['dqs']['adjusted_score'] = adjusted_raw_score

    return qual

def write_dict_to_csv(data_dict, fname):
    fname = os.path.join(glob.OUTPUT_DIR_NAME, fname)
    glob.log.info('going to write dictionary to ' + fname)
    #glob.log.info(json.dumps(data_dict, indent=glob.INDENT_LEVEL))
    with open(fname, 'w') as csv_file:
        #write the header row
        line = '\"country_code\",' #first column is country code
        #now the indicators
        #get all the keys from the first dictionary, they are the same for all dictionaries
        #this is a python3 thing because we would get a dict_keys, we convert it into a list
        #and then the first element of the list we access and then get the keys from there
        #this is specific to the WB data dictionary so this function shouldnt technically be
        #in utils.py, but ok..
        key_list = data_dict[list(data_dict.keys())[0]].keys()

        for k in key_list:
            line = encode_str_in_csv(line, k)

        #remove the last ',' from the end
        line = line[:-1]
        line += '\n'
        #read to write header row
        csv_file.write(line)

        #access dictionary by dictionary or rather country by country
        for k in data_dict.keys():
            #start with an empty line to write
            line = ''
            d = data_dict[k] #this dictionary represents one country
            line = encode_str_in_csv(line, k) #country code
            key_count = 0

            for k2 in d.keys(): #indicators within a country

                val = d[k2]
                if k2 != 'name': #name key already holds a string
                    val = str(val)
                #if there is single quote in the name then rmeove it, caused problem
                #when reading the file back
                val = val.replace('’', '')
                #put the key in quotes, as some keys/values could have spaces
                line = encode_str_in_csv(line, val) #value of individual indicators
                key_count += 1
            glob.log.info('country %s, indicators %d' %(k, key_count))
            #write to csv
            #remove the last ',' from the end
            line = line[:-1]
            line += '\n'
            #glob.log.info(line)
            #ignore any non-ascii characters, this is needed because certain country names
            #have non utf-8 characters..they were causing an exception..
            #line = line.encode('ascii', 'ignore')
            csv_file.write(str(line))

def do_eda(df, filename, ds_name, categorial_feature_list, excluded_list):
    glob.log.info('about to do some EDA (exploratory data analysis) on %s data...' %(ds_name))
    eda_dict = { 'feature': [], 'mean': [], 'mode':[], 'median':[], 'stddev':[]}
    #except for country code all fields are numeric and we can calculate
    #mean, median and sd
    for col in df.columns:
        if col in excluded_list:
            continue
        eda_dict['feature'].append(col)
        if col in categorial_feature_list:
            #calc mode by first storing in a categorical series
            s = pd.Categorical(df[col], categories=df[col].unique())
            cc = s.value_counts()
            pd.Series.sort(cc, inplace=True, ascending=False)
            #what if there are two modes...just handle one case..doesnt happen in our dataset anyway
            #if cc.ix[cc.index[0]] == cc.ix[cc.index[1]]:
            #    mode = cc.index[0] + ',' + cc.index[1]
            #    glob.log.info('there are more than 1 modes for %s[%s]' %(ds_name, col))
            #else:
            #    mode = cc.index[0]
            mode_str = str(cc.index[0]) + ':' + str(cc.ix[cc.index[0]])
            eda_dict['mode'].append(mode_str)
            eda_dict['mean'].append(0)
            eda_dict['median'].append(0)
            eda_dict['stddev'].append(0)
        else:
            #calc other descriptive stats
            eda_dict['mode'].append(0)
            eda_dict['mean'].append(df[col].mean())
            eda_dict['median'].append(df[col].median())
            eda_dict['stddev'].append(np.sqrt(df[col].var()))
    eda_df = pd.DataFrame(eda_dict)
    eda_df.to_csv(filename, index=False, encoding='utf-8')
    try:
        glob.log.info(eda_df)
    except Exception as e:
        glob.log.error('Exception while logging eda_df: ' + str(e))

def detect_outliers(df, ds_name, excluded_col_list, key_col_name):
    glob.log.info('Detecting outliers for %s dataset' %(ds_name))
    fname = os.path.join(glob.OUTPUT_DIR_NAME, glob.EDA_DIR, ds_name + '_' + glob.OUTLIERS_CSV)
    f = open(fname, 'w')
    f.write('dataset,entry,field,value,3rdStdDev\n')
    for col in df.columns:
        if col in excluded_col_list:
            continue
        #refer to the column as a series, just for ease of understanding
        S = df[col]
        # we want to say anything outside of the 3rd standard deviation is an outlier...
        outliers = S[((S-S.mean()).abs()>3*S.std())]
        if len(outliers) > 0:
            #print out the outliers
            sigma = S.std()
            for i in outliers.index:
                entry = df.iloc[i]
                glob.log.error('[%s] for entry %s, field %s has value %f which is outside the 3rd stddev(%f)'
                               %(ds_name, entry[key_col_name], col, entry[col], 3*sigma))
                f.write('"%s","%s","%s","%f","%f"\n' %(ds_name, entry[key_col_name], col, entry[col], 3*sigma))
    f.close()

def calc_r(ds_name, fname, df, feature_list):
    glob.log.info('Calculating r for %s dataset' %(ds_name))
    f = open(fname, 'w')
    f.write('Dataset,feature1,feature2,r\n')
    #remove all NA values as pearsons needs to have both series of the same size
    df2 = df[feature_list].dropna()
    #calculate perason coefficient for all possible combinations
    for i in range(len(feature_list)-1):
        for j in range(i+1, len(feature_list)):
            r = pearsonr(df2[feature_list[i]], df2[feature_list[j]])[0]
            glob.log.info('Pearson coeff(r) for %s and %s is %f' %(feature_list[i], feature_list[j], r))
            f.write('%s,%s,%s,%f\n' %(ds_name,feature_list[i], feature_list[j], r))
    f.close()

def calc_dqs(df):
    df_temp = pd.isnull(df)
    num_cells = df_temp.shape[0]*df_temp.shape[1]
    empty_cells = 0
    for c in df_temp.columns:
        empty_cells += sum(df_temp[c])
    dqs = ((num_cells-empty_cells)*100)/num_cells
    glob.log.info('data quality score for dataset is %f' %(dqs))
    return dqs

def get_numeric_feature_list(df, excluded_feature_list):
    numeric_features = []
    for col in df.columns:
        try:
            x=df[col].iloc[0]
            float(x)#typecast the data to float to test if it is numeric
        except:
            glob.log.info('%s is not a numeric feature, ignoring' %(col))
        else:
            if col not in excluded_feature_list:
                numeric_features.append(col)
    return numeric_features


Homework/02 - Data from the Web/Question 2.py

# coding: utf-8

# Obtain all the data for the Master students, starting from 2007. Compute how many months it took each master student to complete their master, for those that completed it. Partition the data between male and female students, and compute the average -- is the difference in average statistically significant?
#
# Notice that master students' data is more tricky than the bachelors' one, as there are many missing records in the IS-Academia database. Therefore, try to guess how much time a master student spent at EPFL by at least checking the distance in months between Master semestre 1 and Master semestre 2. If the Mineur field is not empty, the student should also appear registered in Master semestre 3. Last but not the least, don't forget to check if the student has an entry also in the Projet Master tables. Once you can handle well this data, compute the "average stay at EPFL" for master students. Now extract all the students with a Spécialisation and compute the "average stay" per each category of that attribute -- compared to the general average, can you find any specialization for which the difference in average is statistically significant?

# In[1]:

# Requests : make http requests to websites
import requests
# BeautifulSoup : parser to manipulate easily html content
from bs4 import BeautifulSoup
# Regular expressions
import re
# Aren't pandas awesome ?
import pandas as pd

# Let's get the first page in which we will be able to extract some interesting content !

# In[2]:

# Ask for the first page on IS Academia. To see it, just type it on your browser address bar : http://isa.epfl.ch/imoniteur_ISAP/!GEDPUBLICREPORTS.filter?ww_i_reportModel=133685247
r = requests.get('http://isa.epfl.ch/imoniteur_ISAP/!GEDPUBLICREPORTS.filter?ww_i_reportModel=133685247')
htmlContent = BeautifulSoup(r.content, 'html.parser')

# In[3]:

print(htmlContent.prettify())

# Now we need to make other requests to IS Academia, which specify every parameter : computer science students, all the years, and all bachelor semester (which are a couple of two values : pedagogic period and semester type). Thus, we're going to get all the parameters we need to make the next request :

# In[4]:

# We first get the "Computer science" value
computerScienceField = htmlContent.find('option', text='Informatique')
computerScienceField

# In[5]:

computerScienceValue = computerScienceField.get('value')
computerScienceValue

# In[6]:

# Then, we're going to need all the academic years values.
academicYearsField = htmlContent.find('select', attrs={'name':'ww_x_PERIODE_ACAD'})
academicYearsSet = academicYearsField.findAll('option')

# Since there are several years to remember, we're storing all of them in a table to use them later
academicYearValues = []
# We'll put the textual content in a table aswell ("Master semestre 1", "Master semestre 2"...)
academicYearContent = []

for option in academicYearsSet:
    value = option.get('value')
    # However, we don't want any "null" value
    if value != 'null':
        academicYearValues.append(value)
        academicYearContent.append(option.text)

# In[7]:

# Now, we have all the academic years that might interest us. We wrangle them a little bit so be able to make request more easily later.
academicYearValues_series = pd.Series(academicYearValues)
academicYearContent_series = pd.Series(academicYearContent)
academicYear_df = pd.concat([academicYearContent_series, academicYearValues_series], axis = 1)
academicYear_df.columns= ['Academic_year', 'Value']
academicYear_df = academicYear_df.sort_values(['Academic_year', 'Value'], ascending=[1, 0])
academicYear_df

# In[8]:

# Then, let's get all the pedagogic periods we need. It's a little bit more complicated here because we need to link the pedagogic period with a season (eg : Bachelor 1 is autumn, Bachelor 2 is spring etc.)
# Thus, we need more than the pedagogic values. For doing some tests to associate them with the right season, we need the actual textual value ("Bachelor semestre 1", "Bachelor semestre 2" etc.)
pedagogicPeriodsField = htmlContent.find('select', attrs={'name':'ww_x_PERIODE_PEDAGO'})
pedagogicPeriodsSet = pedagogicPeriodsField.findAll('option')

# Same as above, we'll store the values in a table
pedagogicPeriodValues = []
# We'll put the textual content in a table aswell ("Master semestre 1", "Master semestre 2"...)
pedagogicPeriodContent = []

for option in pedagogicPeriodsSet:
    value = option.get('value')
    if value != 'null':
        pedagogicPeriodValues.append(value)
        pedagogicPeriodContent.append(option.text)

# In[9]:

# Let's make the values and content meet each other
pedagogicPeriodContent_series = pd.Series(pedagogicPeriodContent)
pedagogicPeriodValues_series = pd.Series(pedagogicPeriodValues)
pedagogicPeriod_df = pd.concat([pedagogicPeriodContent_series, pedagogicPeriodValues_series], axis = 1);
pedagogicPeriod_df.columns = ['Pedagogic_period', 'Value']

# In[10]:

# We keep all semesters related to master students
pedagogicPeriod_df_master = pedagogicPeriod_df[[period.startswith('Master') for period in pedagogicPeriod_df.Pedagogic_period]]
pedagogicPeriod_df_minor = pedagogicPeriod_df[[period.startswith('Mineur') for period in pedagogicPeriod_df.Pedagogic_period]]
pedagogicPeriod_df_project = pedagogicPeriod_df[[period.startswith('Projet Master') for period in pedagogicPeriod_df.Pedagogic_period]]

pedagogicPeriod_df = pd.concat([pedagogicPeriod_df_master, pedagogicPeriod_df_minor, pedagogicPeriod_df_project])
pedagogicPeriod_df

# In[11]:

# Lastly, we need to extract the values associated with autumn and spring semesters.
semesterTypeField = htmlContent.find('select', attrs={'name':'ww_x_HIVERETE'})
semesterTypeSet = semesterTypeField.findAll('option')

# Again, we need to store the values in a table
semesterTypeValues = []
# We'll put the textual content in a table aswell
semesterTypeContent = []

for option in semesterTypeSet:
    value = option.get('value')
    if value != 'null':
        semesterTypeValues.append(value)
        semesterTypeContent.append(option.text)

# In[12]:

# Here are the values for autumn and spring semester :

semesterTypeValues_series = pd.Series(semesterTypeValues)
semesterTypeContent_series = pd.Series(semesterTypeContent)
semesterType_df = pd.concat([semesterTypeContent_series, semesterTypeValues_series], axis = 1)
semesterType_df.columns = ['Semester_type', 'Value']
semesterType_df

# Now, we got all the information to get all the master students !
# Let's make all the requests we need to build our data.
# We will try to do requests such as :
# - Get students from master semester 1 of 2007-2008
# - ...
# - Get students from master semester 4 of 2007-2008
# - Get students from mineur semester 1 of 2007-2008
# - Get students from mineur semester 2 of 2007-2008
# - Get students from master project semester 1 of 2007-2008
# - Get students from master project semester 2 of 2007-2008
#
# ... and so on for each academic year until 2015-2016, the last complete year.
# We can even take the first semester of 2016-2017 into account, to check if some students we though they finished last year are actually still studying. This can be for different reasons : doing a mineur, a project, repeating a semester...

# We can ask for a list of student in two formats : HTML or CSV.
# We choosed to get them in a HTML format because this is the first time that we wrangle data in HTML format, and that may be really useful to learn in order to work with most of the websites in the future !
# The request sent by the browser to IS Academia, to get a list of student in a HTML format, looks like this :
# http://isa.epfl.ch/imoniteur_ISAP/!GEDPUBLICREPORTS.html?arg1=xxx&arg2=yyy
# With "xxx" the value associated with the argument named "arg1", "yyy" the value associated with the argument named "arg2" etc. It uses to have a lot more arguments.
# For instance, we tried to send a request as a "human" through our browser and intercepted it with Postman interceptor.
# We found that the folowing arguments have to be sent :
# ww_x_GPS = -1
# ww_i_reportModel = 133685247
# ww_i_reportModelXsl = 133685270
# ww_x_UNITE_ACAD = 249847 (which is the value of computer science !)
# ww_x_PERIODE_ACAD = X (eg : the value corresponding to 2007-2008 would be 978181)
# ww_x_PERIODE_PEDAGO = Y (eg : 2230106 for Master semestre 1)
# ww_x_HIVERETE = Z (eg : 2936286 for autumn semester)
#
# The last three values X, Y and Z must be replaced with the ones we extracted previously. For instance, if we want to get students from Master, semester 1 (which is necessarily autumn semester) of 2007-2008, the "GET Request" would be the following :
#
# http://isa.epfl.ch/imoniteur_ISAP/!GEDPUBLICREPORTS.html?ww_x_GPS=-1&ww_i_reportModel=133685247&ww_i_reportModelXsl=133685270&ww_x_UNITE_ACAD=249847&ww_x_PERIODE_ACAD=978181&ww_x_PERIODE_PEDAGO=2230106&ww_x_HIVERETE=2936286
#
# So let's cook all the requests we're going to send !

# In[13]:

# Let's put the semester types aside, because we're going to need them
autumn_semester_value = semesterType_df.loc[semesterType_df['Semester_type'] == 'Semestre d\'automne', 'Value']
autumn_semester_value = autumn_semester_value.iloc[0]

spring_semester_value = semesterType_df.loc[semesterType_df['Semester_type'] == 'Semestre de printemps', 'Value']
spring_semester_value = spring_semester_value.iloc[0]

# In[14]:

# Here is the list of the GET requests we will sent to IS Academia
requestsToISAcademia = []

# We'll need to associate all the information associated with the requests to help wrangling data later :
academicYearRequests = []
pedagogicPeriodRequests = []
semesterTypeRequests = []

# Go all over the years ('2007-2008', '2008-2009' and so on)
for academicYear_row in academicYear_df.itertuples(index=True, name='Academic_year'):

    # The year (eg: '2007-2008')
    academicYear = academicYear_row.Academic_year

    # The associated value (eg: '978181')
    academicYear_value = academicYear_row.Value

    # We get all the pedagogic periods associated with this academic year
    for pegagogicPeriod_row in pedagogicPeriod_df.itertuples(index=True, name='Pedagogic_period'):

        # The period (eg: 'Master semestre 1')
        pedagogicPeriod = pegagogicPeriod_row.Pedagogic_period

        # The associated value (eg: '2230106')
        pegagogicPeriod_Value = pegagogicPeriod_row.Value

        # We need to associate the corresponding semester type (eg: Master semester 1 is autumn, but Master semester 2 will be spring)
        if (pedagogicPeriod.endswith('1') or pedagogicPeriod.endswith('3') or pedagogicPeriod.endswith('automne')):
            semester_Value = autumn_semester_value
            semester = 'Autumn'
        else:
            semester_Value = spring_semester_value
            semester = 'Spring'

        # This print line is only for debugging if you want to check something
        # print("academic year = " + academicYear_value + ", pedagogic value = " + pegagogicPeriod_Value + ", pedagogic period is " + pedagogicPeriod + " (semester type value = " + semester_Value + ")")

        # We're ready to cook the request !
        request = 'http://isa.epfl.ch/imoniteur_ISAP/!GEDPUBLICREPORTS.html?ww_x_GPS=-1&ww_i_reportModel=133685247&ww_i_reportModelXsl=133685270&ww_x_UNITE_ACAD=' + computerScienceValue
        request = request + '&ww_x_PERIODE_ACAD=' + academicYear_value
        request = request + '&ww_x_PERIODE_PEDAGO=' + pegagogicPeriod_Value
        request = request + '&ww_x_HIVERETE=' + semester_Value

        # Add the newly created request to our wish list...
        requestsToISAcademia.append(request)
        # And we save the corresponding information for each request
        pedagogicPeriodRequests.append(pedagogicPeriod)
        academicYearRequests.append(academicYear)
        semesterTypeRequests.append(semester)

# In[15]:

# Here is the list of all the requests we have to send !
# requestsToISAcademia

# In[16]:

# Here are the corresponding years for each request
# academicYearRequests

# In[17]:

# Same for associated pedagogic periods
# pedagogicPeriodRequests

# In[18]:

# Last but not the least, the semester types
# semesterTypeRequests

# In[19]:

academicYearRequests_series = pd.Series(academicYearRequests)
pedagogicPeriodRequests_series = pd.Series(pedagogicPeriodRequests)
requestsToISAcademia_series = pd.Series(requestsToISAcademia)

# Let's summarize everything in a dataframe...
requests_df = pd.concat([academicYearRequests_series, pedagogicPeriodRequests_series, requestsToISAcademia_series], axis = 1)
requests_df.columns = ['Academic_year', 'Pedagogic_period', 'Request']

requests_df

# In[ ]:

# The requests are now ready to be sent to IS Academia. Let's try it out !

# TIME OUT : We stopped right here for our homework. What is below should look like the beginning of a loop that gets students lists from IS Academia. It's not finished at all :(

# In[20]:

# WARNING : NEXT LINE IS COMMENTED FOR DEBGUGGING THE FIRST REQUEST ONLY. UNCOMMENT IT AND INDENT THE CODE CORRECTLY TO MAKE ALL THE REQUESTS

#for request in requestsToISAcademia: # LINE TO UNCOMMENT TO SEND ALL REQUESTS
request = requestsToISAcademia[0] # LINE TO COMMENT TO SEND ALL REQUESTS
print(request)

# Send the request to IS Academia
r = requests.get(request)

# Here is the HTML content of IS Academia's response
htmlContent = BeautifulSoup(r.content, 'html.parser')

# Let's extract some data...
computerScienceField = htmlContent.find('option', text='Informatique')

# In[21]:

# Getting the table of students
# Let's make the columns
columns = []
table = htmlContent.find('table')
th = table.find('th', text='Civilité')
columns.append(th.text)
# Go through the table until the last column
while th.findNext('').name == 'th':
    th = th.findNext('')
    columns.append(th.text)

# This array will contain all the students
studentsTable = []

# DON'T RUN THE NEXT CELL OR IT WILL CRASH ! :x

# In[22]:

# Getting the information about the student we're "looping on"
currentStudent = []
tr = th.findNext('tr')
children = tr.children
for child in children:
    currentStudent.append(child.text)

# Add the student to the array
studentsTable.append(currentStudent)

# In[23]:

a = tr.findNext('tr')
a

# In[ ]:

while tr.findNext('tr') is not None:
    tr = th.findNext('tr')
    children = tr.children
    for child in children:
        currentStudent.append(child.text)
    studentsTable.append(currentStudent)

studentsTable

# In[ ]:

#tr = th.parent
#td = th.findNext('td')
#td.text
#th.findNext('th')
#th.findNext('th')
#tr = tr.findNext('tr')
#tr

# In[ ]:

print(htmlContent.prettify())


from __future__ import absolute_import, division, print_function

import pytest
paramiko = pytest.importorskip('paramiko')

import pandas as pd
import numpy as np
import re
import os
import sys

from odo.utils import tmpfile, filetext
from odo.directory import _Directory, Directory
from odo.backends.ssh import SSH, resource, ssh_pattern, sftp, drop, connect
from odo.backends.csv import CSV
from odo import into, discover, CSV, JSONLines, JSON, convert
from odo.temp import _Temp, Temp
from odo.compatibility import ON_TRAVIS_CI
import socket

skipif = pytest.mark.skipif

try:
    ssh = connect(hostname='localhost')
    ssh.close()
except socket.error:
    pytest.skip('Could not connect')
except paramiko.PasswordRequiredException as e:
    pytest.skip(str(e))
except paramiko.SSHException as e:
    pytest.skip(str(e))

def test_resource():
    r = resource('ssh://joe@localhost:/path/to/myfile.csv')
    assert isinstance(r, SSH(CSV))
    assert r.path == '/path/to/myfile.csv'
    assert r.auth['hostname'] == 'localhost'
    assert r.auth['username'] == 'joe'

def test_connect():
    a = connect(hostname='localhost')
    b = connect(hostname='localhost')
    assert a is b

    a.close()

    c = connect(hostname='localhost')
    assert a is c
    assert c.get_transport() and c.get_transport().is_active()

def test_resource_directory():
    r = resource('ssh://joe@localhost:/path/to/')
    assert issubclass(r.subtype, _Directory)

    r = resource('ssh://joe@localhost:/path/to/*.csv')
    assert r.subtype == Directory(CSV)
    assert r.path == '/path/to/'

def test_discover():
    with filetext('name,balance\nAlice,100\nBob,200') as fn:
        local = CSV(fn)
        remote = SSH(CSV)(fn, hostname='localhost')

        assert discover(local) == discover(remote)

def test_discover_from_resource():
    with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
        local = CSV(fn)
        remote = resource('ssh://localhost:' + fn)

        assert discover(local) == discover(remote)

def test_ssh_pattern():
    uris = ['localhost:myfile.csv',
            '127.0.0.1:/myfile.csv',
            'user@127.0.0.1:/myfile.csv',
            'user@127.0.0.1:/*.csv',
            'user@127.0.0.1:/my-dir/my-file3.csv']
    for uri in uris:
        assert re.match(ssh_pattern, uri)

def test_copy_remote_csv():
    with tmpfile('csv') as target:
        with filetext('name,balance\nAlice,100\nBob,200',
                      extension='csv') as fn:
            csv = resource(fn)

            uri = 'ssh://localhost:%s.csv' % target
            scsv = into(uri, csv)

            assert isinstance(scsv, SSH(CSV))
            assert discover(scsv) == discover(csv)

            # Round trip
            csv2 = into(target, scsv)
            assert into(list, csv) == into(list, csv2)

def test_drop():
    with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
        with tmpfile('csv') as target:
            scsv = SSH(CSV)(target, hostname='localhost')

            assert not os.path.exists(target)

            conn = sftp(**scsv.auth)
            conn.put(fn, target)

            assert os.path.exists(target)

            drop(scsv)
            drop(scsv)

            assert not os.path.exists(target)

def test_drop_of_csv_json_lines_use_ssh_version():
    from odo.backends.ssh import drop_ssh
    for typ in [CSV, JSON, JSONLines]:
        assert drop.dispatch(SSH(typ)) == drop_ssh

def test_convert_local_file_to_temp_ssh_file():
    with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
        csv = CSV(fn)
        scsv = convert(Temp(SSH(CSV)), csv, hostname='localhost')

        assert into(list, csv) == into(list, scsv)

@skipif(ON_TRAVIS_CI, reason="Don't know")
def test_temp_ssh_files():
    with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
        csv = CSV(fn)
        scsv = into(Temp(SSH(CSV)), csv, hostname='localhost')
        assert discover(csv) == discover(scsv)

        assert isinstance(scsv, _Temp)

@skipif(ON_TRAVIS_CI, reason="Don't know")
def test_convert_through_temporary_local_storage():
    with filetext('name,quantity\nAlice,100\nBob,200', extension='csv') as fn:
        csv = CSV(fn)
        df = into(pd.DataFrame, csv)
        scsv = into(Temp(SSH(CSV)), csv, hostname='localhost')

        assert into(list, csv) == into(list, scsv)

        scsv2 = into(Temp(SSH(CSV)), df, hostname='localhost')
        assert into(list, scsv2) == into(list, df)

        sjson = into(Temp(SSH(JSONLines)), df, hostname='localhost')
        assert (into(np.ndarray, sjson) == into(np.ndarray, df)).all()

@skipif(ON_TRAVIS_CI and sys.version_info[0] == 3,
        reason='Strange hanging on travis for python33 and python34')
def test_ssh_csv_to_s3_csv():
    # for some reason this can only be run in the same file as other ssh tests
    # and must be a Temp(SSH(CSV)) otherwise tests above this one fail
    s3_bucket = pytest.importorskip('odo.backends.tests.test_aws').s3_bucket

    with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
        remote = into(Temp(SSH(CSV)), CSV(fn), hostname='localhost')
        with s3_bucket('.csv') as b:
            result = into(b, remote)
            assert discover(result) == discover(resource(b))

@skipif(ON_TRAVIS_CI and sys.version_info[0] == 3,
        reason='Strange hanging on travis for python33 and python34')
def test_s3_to_ssh():
    pytest.importorskip('boto')

    tips_uri = 's3://nyqpug/tips.csv'
    with tmpfile('.csv') as fn:
        result = into(Temp(SSH(CSV))(fn, hostname='localhost'), tips_uri)
        assert into(list, result) == into(list, tips_uri)
        assert discover(result) == discover(resource(tips_uri))


"""
========================================
Label Propagation digits active learning
========================================

Demonstrates an active learning technique to learn handwritten digits
using label propagation.

We start by training a label propagation model with only 10 labeled points,
then we select the top five most uncertain points to label. Next, we train
with 15 labeled points (original 10 + 5 new ones). We repeat this process
four times to have a model trained with 30 labeled examples.

A plot will appear showing the top 5 most uncertain digits for each iteration
of training. These may or may not contain mistakes, but we will train the next
model with their true labels.
"""
print(__doc__)

# Authors: Clay Woolam <clay@woolam.org>
# Licence: BSD

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn import datasets
from sklearn.semi_supervised import label_propagation
from sklearn.metrics import classification_report, confusion_matrix

digits = datasets.load_digits()
rng = np.random.RandomState(0)
indices = np.arange(len(digits.data))
rng.shuffle(indices)

X = digits.data[indices[:330]]
y = digits.target[indices[:330]]
images = digits.images[indices[:330]]

n_total_samples = len(y)
n_labeled_points = 10

unlabeled_indices = np.arange(n_total_samples)[n_labeled_points:]
f = plt.figure()

for i in range(5):
    y_train = np.copy(y)
    y_train[unlabeled_indices] = -1

    lp_model = label_propagation.LabelSpreading(gamma=0.25, max_iter=5)
    lp_model.fit(X, y_train)

    predicted_labels = lp_model.transduction_[unlabeled_indices]
    true_labels = y[unlabeled_indices]

    cm = confusion_matrix(true_labels, predicted_labels,
                          labels=lp_model.classes_)

    print('Iteration %i %s' % (i, 70 * '_'))
    print("Label Spreading model: %d labeled & %d unlabeled (%d total)"
          % (n_labeled_points, n_total_samples - n_labeled_points, n_total_samples))

    print(classification_report(true_labels, predicted_labels))

    print("Confusion matrix")
    print(cm)

    # compute the entropies of transduced label distributions
    pred_entropies = stats.distributions.entropy(
        lp_model.label_distributions_.T)

    # select five digit examples that the classifier is most uncertain about
    uncertainty_index = uncertainty_index = np.argsort(pred_entropies)[-5:]

    # keep track of indices that we get labels for
    delete_indices = np.array([])

    f.text(.05, (1 - (i + 1) * .183),
           "model %d\n\nfit with\n%d labels" % ((i + 1), i * 5 + 10), size=10)
    for index, image_index in enumerate(uncertainty_index):
        image = images[image_index]

        sub = f.add_subplot(5, 5, index + 1 + (5 * i))
        sub.imshow(image, cmap=plt.cm.gray_r)
        sub.set_title('predict: %i\ntrue: %i' % (
            lp_model.transduction_[image_index], y[image_index]), size=10)
        sub.axis('off')

        # labeling 5 points, remote from labeled set
        delete_index, = np.where(unlabeled_indices == image_index)
        delete_indices = np.concatenate((delete_indices, delete_index))

    unlabeled_indices = np.delete(unlabeled_indices, delete_indices)
    n_labeled_points += 5

f.suptitle("Active learning with Label Propagation.\nRows show 5 most "
           "uncertain labels to learn with the next model.")
plt.subplots_adjust(0.12, 0.03, 0.9, 0.8, 0.2, 0.45)
plt.show()


import sys
import os
import pyflann
import params
import numpy as np
import draw_func2 as df2
import helpers
np.random.seed(5)
# Parameters
tdim   =   2; # Target viewing dimensions
dim    =   2; # Calculation dimension
if len(sys.argv) == 2:
    tdim = int(sys.argv[1])
    dim  = int(sys.argv[1])
K      =   4;
checks = 128;
nQuery =   8;
nData  = 1024;

# Script
def quick_flann_index(data):
    data_flann = pyflann.FLANN()
    flann_params =  params.VSMANY_FLANN_PARAMS
    checks = flann_params['checks']
    data_flann.build_index(data, **flann_params)
    return data_flann

def reciprocal_nearest_neighbors(query, data, data_flann, checks):
    nQuery, dim = query.shape
    # Assign query features to K nearest database features
    (qfx2_dx, qfx2_dists) = data_flann.nn_index(query, K, checks=checks)
    # Assign those nearest neighbors to K nearest database features
    qx2_nn = data[qfx2_dx]
    qx2_nn.shape = (nQuery*K, dim)
    (_nn2_dx, nn2_dists) = data_flann.nn_index(qx2_nn, K, checks=checks)
    # Get the maximum distance of the reciprocal neighbors
    nn2_dists.shape = (nQuery, K, K)
    qfx2_maxdist = nn2_dists.max(2)
    # Test if nearest neighbor distance is less than reciprocal distance
    isReciprocal = qfx2_dists < qfx2_maxdist
    return qfx2_dx, qfx2_dists, isReciprocal

data  = np.random.rand(nData,  dim)
query = np.random.rand(nQuery, dim)

nQuery = len(query)
# Find query's Nearest Neighbors in data
data_flann = quick_flann_index(data)
(qfx2_dx, qfx2_dists) = data_flann.nn_index(query, K, checks=checks)
qx2_nn = data[qfx2_dx]
# get k-reciprocal nearest neighbors max distance
qx2_nn.shape = (nQuery*K, dim)
(nn2_dx, nn2_dists) = data_flann.nn_index(qx2_nn, K, checks=checks)
nn2_data = data[nn2_dx] # data's nearest neighbors
nn2_dists.shape = (nQuery, K, K)
qx2_nn.shape = (nQuery, K, dim)
qfx2_maxdist = nn2_dists.max(2)
# A neighbor is a K reciprocal if you are within the
# max distance of the assigned points K nearest neighbors
isReciprocal = qfx2_dists < qfx2_maxdist
krx2_nn  = qx2_nn[isReciprocal]
krx2_qfx = helpers.tiled_range(nQuery, K)[isReciprocal]
krx2_query = query[krx2_qfx]

# Enforce viewable dimensionality
if dim != tdim:
    import sklearn.decomposition
    print('Plotting pca.transform dimensionality')
    pca = sklearn.decomposition.PCA(copy=True, n_components=tdim, whiten=False)
    pca.fit(data)
    query_  = pca.transform(query)
    data_   = pca.transform(data)
    nn2_data_ = pca.transform(nn2_data)
    qx2_nn_ = pca.transform(qx2_nn)
    krx2_query_ = pca.transform(krx2_query)
    krx2_nn_ = pca.transform(krx2_nn)
else:
    print('Plotting full dimensionality')
    query_  = (query)
    data_   = (data)
    qx2_nn_ = (qx2_nn)
    krx2_query_ = (krx2_query)
    krx2_nn_ = (krx2_nn)

# Figure and Axis
plt = df2.plt
df2.reset()
fig = plt.figure(1)
if tdim == 2:
    ax  = fig.add_subplot(111)
elif tdim > 2:
    from mpl_toolkits.mplot3d import Axes3D
    ax  = fig.add_subplot(111, projection='3d')

def plot_points(data, color, marker):
    dataT = data.T
    if len(dataT) == 2:
        ax.plot(dataT[0], dataT[1], color=color, marker=marker, linestyle='None')
    elif len(dataT) == 3:
        ax.scatter(dataT[0], dataT[1], dataT[2], color=color, marker=marker)

def plot_lines(point_pairs, color):
    for pair in point_pairs:
        dataT = pair.T
        if len(dataT) == 2:
            ax.plot(dataT[0], dataT[1], color=color)
        elif len(dataT) == 3:
            ax.plot(dataT[0], dataT[1], dataT[2], color=color)
            #plt.scatter(dataT[0], dataT[1], dataT[2], s=20, color=color)

# Plot query / data
plot_points(data_, 'b', 'x')
plot_points(query_,'b', 'o')
# Plot KNN
qx2_nn_.shape = (nQuery, K, tdim)
point_pairs = [np.vstack((query_[qx], qx2_nn_[qx,k])) for qx in xrange(nQuery) for k in xrange(K)]
plot_lines(point_pairs, (1, 0, 0, .8))
# Plot NN's KNN
qx2_nn_.shape = (nQuery*K, tdim)
nRes = len(qx2_nn_)
point_pairs3 = [np.vstack((qx2_nn_[nnx], nn2_data_[nnx,k])) for nnx in xrange(nRes) for k in xrange(K)]
plot_lines(point_pairs3, (1, .8, .8, .5))

# Plot KRNN
point_pairs2 = map(np.vstack, zip(krx2_query_, krx2_nn_))
plot_lines(point_pairs2, (0, 1, 0, .9))
df2.update()
# Show
df2.set_figtitle('KRNN=(Green), NN=(Red), NNR=(Pink), dims=%r, K=%r' % (dim, K))
exec(df2.present())


"""
====================================================================
Comparison of the K-Means and MiniBatchKMeans clustering algorithms
====================================================================

We want to compare the performance of the MiniBatchKMeans and KMeans:
the MiniBatchKMeans is faster, but gives slightly different results (see
:ref:`mini_batch_kmeans`).

We will cluster a set of data, first with KMeans and then with
MiniBatchKMeans, and plot the results.
We will also plot the points that are labelled differently between the two
algorithms.
"""
print(__doc__)

import time

import numpy as np
import matplotlib.pyplot as plt

from sklearn.cluster import MiniBatchKMeans, KMeans
from sklearn.metrics.pairwise import pairwise_distances_argmin
from sklearn.datasets.samples_generator import make_blobs

##############################################################################
# Generate sample data
np.random.seed(0)

batch_size = 45
centers = [[1, 1], [-1, -1], [1, -1]]
n_clusters = len(centers)
X, labels_true = make_blobs(n_samples=3000, centers=centers, cluster_std=0.7)

##############################################################################
# Compute clustering with Means

k_means = KMeans(init='k-means++', n_clusters=3, n_init=10)
t0 = time.time()
k_means.fit(X)
t_batch = time.time() - t0
k_means_labels = k_means.labels_
k_means_cluster_centers = k_means.cluster_centers_
k_means_labels_unique = np.unique(k_means_labels)

##############################################################################
# Compute clustering with MiniBatchKMeans

mbk = MiniBatchKMeans(init='k-means++', n_clusters=3, batch_size=batch_size,
                      n_init=10, max_no_improvement=10, verbose=0)
t0 = time.time()
mbk.fit(X)
t_mini_batch = time.time() - t0
mbk_means_labels = mbk.labels_
mbk_means_cluster_centers = mbk.cluster_centers_
mbk_means_labels_unique = np.unique(mbk_means_labels)

##############################################################################
# Plot result

fig = plt.figure(figsize=(8, 3))
fig.subplots_adjust(left=0.02, right=0.98, bottom=0.05, top=0.9)
colors = ['#4EACC5', '#FF9C34', '#4E9A06']

# We want to have the same colors for the same cluster from the
# MiniBatchKMeans and the KMeans algorithm. Let's pair the cluster centers per
# closest one.

order = pairwise_distances_argmin(k_means_cluster_centers,
                                  mbk_means_cluster_centers)

# KMeans
ax = fig.add_subplot(1, 3, 1)
for k, col in zip(range(n_clusters), colors):
    my_members = k_means_labels == k
    cluster_center = k_means_cluster_centers[k]
    ax.plot(X[my_members, 0], X[my_members, 1], 'w',
            markerfacecolor=col, marker='.')
    ax.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
            markeredgecolor='k', markersize=6)
ax.set_title('KMeans')
ax.set_xticks(())
ax.set_yticks(())
plt.text(-3.5, 1.8,  'train time: %.2fs\ninertia: %f' % (
    t_batch, k_means.inertia_))

# MiniBatchKMeans
ax = fig.add_subplot(1, 3, 2)
for k, col in zip(range(n_clusters), colors):
    my_members = mbk_means_labels == order[k]
    cluster_center = mbk_means_cluster_centers[order[k]]
    ax.plot(X[my_members, 0], X[my_members, 1], 'w',
            markerfacecolor=col, marker='.')
    ax.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
            markeredgecolor='k', markersize=6)
ax.set_title('MiniBatchKMeans')
ax.set_xticks(())
ax.set_yticks(())
plt.text(-3.5, 1.8, 'train time: %.2fs\ninertia: %f' %
         (t_mini_batch, mbk.inertia_))

# Initialise the different array to all False
different = (mbk_means_labels == 4)
ax = fig.add_subplot(1, 3, 3)

for l in range(n_clusters):
    different += ((k_means_labels == k) != (mbk_means_labels == order[k]))

identic = np.logical_not(different)
ax.plot(X[identic, 0], X[identic, 1], 'w',
        markerfacecolor='#bbbbbb', marker='.')
ax.plot(X[different, 0], X[different, 1], 'w',
        markerfacecolor='m', marker='.')
ax.set_title('Difference')
ax.set_xticks(())
ax.set_yticks(())

plt.show()


#!/usr/bin/env python
# ----------------------------------------------------------------------------
# Copyright 2015 Nervana Systems Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------

"""
Convolutional autoencoder example network for MNIST data set
"""

import numpy as np

from neon.data import ArrayIterator, load_mnist
from neon.initializers import Uniform
from neon.layers import Conv, Pooling, GeneralizedCost, Deconv
from neon.models import Model
from neon.optimizers import GradientDescentMomentum
from neon.transforms import Rectlin, SumSquared
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser

# parse the command line arguments
parser = NeonArgparser(__doc__)
args = parser.parse_args()

# Load dataset
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir)

# Set input and target to X_train
train = ArrayIterator(X_train, lshape=(1, 28, 28))

# Initialize the weights and the learning rule
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.001, momentum_coef=0.9)

# Define the layers
layers = [Conv((4, 4, 8), init=init_uni, activation=Rectlin()),
          Pooling(2),
          Conv((4, 4, 32), init=init_uni, activation=Rectlin()),
          Pooling(2),
          Deconv(fshape=(4, 4, 8), init=init_uni, activation=Rectlin()),
          Deconv(fshape=(3, 3, 8), init=init_uni, activation=Rectlin(), strides=2),
          Deconv(fshape=(2, 2, 1), init=init_uni, strides=2, padding=1)]

# Define the cost
cost = GeneralizedCost(costfunc=SumSquared())

model = Model(layers=layers)

# configure callbacks
callbacks = Callbacks(model, **args.callback_args)

# Fit the model
model.fit(train, optimizer=opt_gdm, num_epochs=args.epochs, cost=cost, callbacks=callbacks)

# Plot the reconstructed digits
try:
    from matplotlib import pyplot, cm
    fi = 0
    nrows = 10
    ncols = 12
    test = np.zeros((28*nrows, 28*ncols))
    idxs = [(row, col) for row in range(nrows) for col in range(ncols)]
    for row, col in idxs:
        im = model.layers.layers[-1].outputs.get()[:, fi].reshape((28, 28))
        test[28*row:28*(row+1):, 28*col:28*(col+1)] = im
        fi = fi + 1
    pyplot.matshow(test, cmap=cm.gray)
    pyplot.savefig('Reconstructed.png')
except ImportError:
    print 'matplotlib needs to be manually installed to generate plots'


from __future__ import (absolute_import, division, print_function)
from PyQt5 import QtCore, QtWidgets
import sys
import numpy as np
from layer import Layer, MSLD

from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
import matplotlib.cm
from matplotlib.patches import Polygon
from matplotlib.collections import PatchCollection

class layerplot(QtWidgets.QWidget):
    def __init__(self, *args):
        QtWidgets.QWidget.__init__(self, *args)
        sample=[Layer(nsld=5),Layer(thickness=2.,nsld=3),Layer(nsld=5),Layer(nsld=4.,thickness=np.inf)]
        self.m = PlotCanvas(sample, self)
        self.m.move(0,0)

    def resizeEvent(self, event):
        self.m.setGeometry(self.rect())

class PlotCanvas(FigureCanvas):

    def __init__(self, layers, parent=None):
        self.fig = Figure()
        self.axes = self.fig.add_subplot(111)
        FigureCanvas.__init__(self, self.fig)
        self.setParent(parent)
        FigureCanvas.setSizePolicy(self,
                QtWidgets.QSizePolicy.Expanding,
                QtWidgets.QSizePolicy.Expanding)
        FigureCanvas.updateGeometry(self)
        self.data=layers
        self.variable='nsld'
        self.plot()
        self.fig.canvas.mpl_connect('pick_event', self.onpick)

    def onpick(self,event):
        ind=event.ind[0]
        if ind==len(self.data)-1:
            ind='substrate'
        print('picked layer {0}'.format(ind))
        return True

    def plot(self):
        layer_thick_array=np.array([l.thickness for l in self.data])
        layer_nsld_array =np.array([l.nsld for l in self.data])
        depth=np.zeros(len(layer_thick_array))
        depth[1:]=layer_thick_array.cumsum()[:-1]
        patches=[]
        N=len(self.data)
        for i in range(N-1):
            polygon=Polygon([[depth[i],0.],[depth[i],layer_nsld_array[i]],[depth[i+1],layer_nsld_array[i]],[depth[i+1],0]],True)
            patches.append(polygon)
        polygon=Polygon([[depth[N-1],0.],[depth[N-1],layer_nsld_array[N-1]],[depth[N-1]+1,layer_nsld_array[N-1]],[depth[N-1]+1,0]],True)
        patches.append(polygon)
        p = PatchCollection(patches, cmap=matplotlib.cm.jet, alpha=0.4, picker=True)
        colors = 100*np.random.rand(len(patches))
        p.set_array(np.array(colors))
        ax = self.figure.add_subplot(111)
        ax.add_collection(p)
        ax.set_title('NSLD')
        ax.set_xlim(np.array([0,depth[-1]])*1.2)
        ax.set_ylim(np.array([0,layer_nsld_array.max()])*1.2) #TODO allow negative
        ax.set_xlabel('Thickness')
        ax.set_ylabel('NSLD')
        self.draw()

if __name__=='__main__':
    app=QtWidgets.QApplication(sys.argv)
    mainForm=layerplot()
    mainForm.show()
    sys.exit(app.exec_())