"""
Plot and save key steps in pulses.py for visualizing the alorithm.
"""

import glob
import numpy as np
from scipy import stats
from matplotlib import rcParams, gridspec, ticker
import matplotlib.pyplot as plt
try:
    from matplotlib.colors import colorConverter as cc
except ImportError:
    import matplotlib.colors as cc
try:
    from matplotlib.colors import to_hex
except ImportError:
    from matplotlib.colors import rgb2hex as to_hex
from matplotlib.patches import ConnectionPatch, Rectangle
from matplotlib.lines import Line2D
import warnings
def warn(*args, **kwargs):
    """
    Ignore all warnings.
    """
    pass
warnings.warn=warn

# plotting parameters and colors:
rcParams['font.family'] = 'monospace'
cmap = plt.get_cmap("Dark2")
c_g = cmap(0)
c_o = cmap(1)
c_grey = cmap(7)
cmap_pts = [cmap(2), cmap(3)]

def darker(color, saturation):
    """ Make a color darker.

    From bendalab/plottools package.

    Parameters
    ----------
    color: dict or matplotlib color spec
        A matplotlib color (hex string, name color string, rgb tuple)
        or a dictionary with an 'color' or 'facecolor' key.
    saturation: float
        The smaller the saturation, the darker the returned color.
        A saturation of 0 returns black.
        A saturation of 1 leaves the color untouched.
        A saturation of 2 returns white.

    Returns
    -------
    color: string or dictionary
        The darker color as a hexadecimal RGB string (e.g. '#rrggbb').
        If `color` is a dictionary, a copy of the dictionary is returned
        with the value of 'color' or 'facecolor' set to the darker color.
    """
    try:
        c = color['color']
        cd = dict(**color)
        cd['color'] = darker(c, saturation)
        return cd
    except (KeyError, TypeError):
        try:
            c = color['facecolor']
            cd = dict(**color)
            cd['facecolor'] = darker(c, saturation)
            return cd
        except (KeyError, TypeError):
            if saturation > 2:
                sauration = 2
            if saturation > 1:
                return lighter(color, 2.0-saturation)
            if saturation < 0:
                saturation = 0
            r, g, b = cc.to_rgb(color)
            rd = r*saturation
            gd = g*saturation
            bd = b*saturation
            return to_hex((rd, gd, bd)).upper()

def lighter(color, lightness):
    """Make a color lighter

    From bendalab/plottools package.

    Parameters
    ----------
    color: dict or matplotlib color spec
        A matplotlib color (hex string, name color string, rgb tuple)
        or a dictionary with an 'color' or 'facecolor' key.
    lightness: float
        The smaller the lightness, the lighter the returned color.
        A lightness of 0 returns white.
        A lightness of 1 leaves the color untouched.
        A lightness of 2 returns black.

    Returns
    -------
    color: string or dict
        The lighter color as a hexadecimal RGB string (e.g. '#rrggbb').
        If `color` is a dictionary, a copy of the dictionary is returned
        with the value of 'color' or 'facecolor' set to the lighter color.
    """
    try:
        c = color['color']
        cd = dict(**color)
        cd['color'] = lighter(c, lightness)
        return cd
    except (KeyError, TypeError):
        try:
            c = color['facecolor']
            cd = dict(**color)
            cd['facecolor'] = lighter(c, lightness)
            return cd
        except (KeyError, TypeError):
            if lightness > 2:
                lightness = 2
            if lightness > 1:
                return darker(color, 2.0-lightness)
            if lightness < 0:
                lightness = 0
            r, g, b = cc.to_rgb(color)
            rl = r + (1.0-lightness)*(1.0 - r)
            gl = g + (1.0-lightness)*(1.0 - g)
            bl = b + (1.0-lightness)*(1.0 - b)
            return to_hex((rl, gl, bl)).upper()

def xscalebar(ax, x, y, width, wunit=None, wformat=None, ha='left', va='bottom',
              lw=None, color=None, capsize=None, clw=None, **kwargs):
    """Horizontal scale bar with label.

    From bendalab/plottools package.

    Parameters
    ----------
    ax: matplotlib axes
        Axes where to draw the scale bar.
    x: float
        x-coordinate where to draw the scale bar in relative units of the axes.
    y: float
        y-coordinate where to draw the scale bar in relative units of the axes.
    width: float
        Length of the scale bar in units of the data's x-values.
    wunit: string or None
        Optional unit of the data's x-values.
    wformat: string or None
        Optional format string for formatting the label of the scale bar
        or simply a string used for labeling the scale bar.
    ha: 'left', 'right', or 'center'
        Scale bar aligned left, right, or centered to (x, y)
    va: 'top' or 'bottom'
        Label of the scale bar either above or below the scale bar.
    lw: int, float, None
        Line width of the scale bar.
    color: matplotlib color
        Color of the scalebar.
    capsize: float or None
        If larger then zero draw cap lines at the ends of the bar.
        The length of the lines is given in points (same unit as linewidth).
    clw: int, float, None
        Line width of the cap lines.
    kwargs: key-word arguments
        Passed on to `ax.text()` used to print the scale bar label.
    """
    ax.autoscale(False)
    # ax dimensions:
    pixelx = np.abs(np.diff(ax.get_window_extent().get_points()[:,0]))[0]
    pixely = np.abs(np.diff(ax.get_window_extent().get_points()[:,1]))[0]
    xmin, xmax = ax.get_xlim()
    ymin, ymax = ax.get_ylim()
    unitx = xmax - xmin
    unity = ymax - ymin
    dxu = np.abs(unitx)/pixelx
    dyu = np.abs(unity)/pixely
    # transform x, y from relative units to axis units:
    x = xmin + x*unitx
    y = ymin + y*unity
    # bar length:
    if wformat is None:
        wformat = '%.0f'
        if width < 1.0:
            wformat = '%.1f'
    try:
        ls = wformat % width
        width = float(ls)
    except TypeError:
        ls = wformat
    # bar:
    if ha == 'left':
        x0 = x
        x1 = x+width
    elif ha == 'right':
        x0 = x-width
        x1 = x
    else:
        x0 = x-0.5*width
        x1 = x+0.5*width
    # line width:
    if lw is None:
        lw = 2
    # color:
    if color is None:
        color = 'k'
    # scalebar:
    lh = ax.plot([x0, x1], [y, y], '-', color=color, lw=lw,
                 solid_capstyle='butt', clip_on=False)
    # get y position of line in figure pixel coordinates:
    ly = np.array(lh[0].get_window_extent(ax.get_figure().canvas.get_renderer()))[0,1]
    # caps:
    if capsize is None:
        capsize = 0
    if clw is None:
        clw = 0.5
    if capsize > 0.0:
        dy = capsize*dyu
        ax.plot([x0, x0], [y-dy, y+dy], '-', color=color, lw=clw,
                solid_capstyle='butt', clip_on=False)
        ax.plot([x1, x1], [y-dy, y+dy], '-', color=color, lw=clw,
                solid_capstyle='butt', clip_on=False)
    # label:
    if wunit:
        ls += u'\u2009%s' % wunit
    if va == 'top':
        th = ax.text(0.5*(x0+x1), y, ls, clip_on=False,
                     ha='center', va='bottom', **kwargs)
        # get y coordinate of text bottom in figure pixel coordinates:
        ty = np.array(th.get_window_extent(ax.get_figure().canvas.get_renderer()))[0,1]
        dty = ly+0.5*lw + 2.0 - ty
    else:
        th = ax.text(0.5*(x0+x1), y, ls, clip_on=False,
                     ha='center', va='top', **kwargs)
        # get y coordinate of text bottom in figure pixel coordinates:
        ty = np.array(th.get_window_extent(ax.get_figure().canvas.get_renderer()))[1,1]
        dty = ly-0.5*lw - 2.0 - ty
    th.set_position((0.5*(x0+x1), y+dyu*dty))
    return x0, x1, y

def yscalebar(ax, x, y, height, hunit=None, hformat=None, ha='left', va='bottom',
              lw=None, color=None, capsize=None, clw=None, **kwargs):

    """Vertical scale bar with label.

    From bendalab/plottools package.

    Parameters
    ----------
    ax: matplotlib axes
        Axes where to draw the scale bar.
    x: float
        x-coordinate where to draw the scale bar in relative units of the axes.
    y: float
        y-coordinate where to draw the scale bar in relative units of the axes.
    height: float
        Length of the scale bar in units of the data's y-values.
    hunit: string
        Unit of the data's y-values.
    hformat: string or None
        Optional format string for formatting the label of the scale bar
        or simply a string used for labeling the scale bar.
    ha: 'left' or 'right'
        Label of the scale bar either to the left or to the right
        of the scale bar.
    va: 'top', 'bottom', or 'center'
        Scale bar aligned above, below, or centered on (x, y).
    lw: int, float, None
        Line width of the scale bar.
    color: matplotlib color
        Color of the scalebar.
    capsize: float or None
        If larger then zero draw cap lines at the ends of the bar.
        The length of the lines is given in points (same unit as linewidth).
    clw: int, float
        Line width of the cap lines.
    kwargs: key-word arguments
        Passed on to `ax.text()` used to print the scale bar label.
    """

    ax.autoscale(False)
    # ax dimensions:
    pixelx = np.abs(np.diff(ax.get_window_extent().get_points()[:,0]))[0]
    pixely = np.abs(np.diff(ax.get_window_extent().get_points()[:,1]))[0]
    xmin, xmax = ax.get_xlim()
    ymin, ymax = ax.get_ylim()
    unitx = xmax - xmin
    unity = ymax - ymin
    dxu = np.abs(unitx)/pixelx
    dyu = np.abs(unity)/pixely
    # transform x, y from relative units to axis units:
    x = xmin + x*unitx
    y = ymin + y*unity
    # bar length:
    if hformat is None:
        hformat = '%.0f'
        if height < 1.0:
            hformat = '%.1f'
    try:
        ls = hformat % height
        width = float(ls)
    except TypeError:
        ls = hformat
    # bar:
    if va == 'bottom':
        y0 = y
        y1 = y+height
    elif va == 'top':
        y0 = y-height
        y1 = y
    else:
        y0 = y-0.5*height
        y1 = y+0.5*height
    # line width:
    if lw is None:
        lw = 2
    # color:
    if color is None:
        color = 'k'
    # scalebar:
    lh = ax.plot([x, x], [y0, y1], '-', color=color, lw=lw,
                 solid_capstyle='butt', clip_on=False)
    # get x position of line in figure pixel coordinates:
    lx = np.array(lh[0].get_window_extent(ax.get_figure().canvas.get_renderer()))[0,0]
    # caps:
    if capsize is None:
        capsize = 0
    if clw is None:
        clw = 0.5
    if capsize > 0.0:
        dx = capsize*dxu
        ax.plot([x-dx, x+dx], [y0, y0], '-', color=color, lw=clw, solid_capstyle='butt',
                clip_on=False)
        ax.plot([x-dx, x+dx], [y1, y1], '-', color=color, lw=clw, solid_capstyle='butt',
                clip_on=False)
    # label:
    if hunit:
        ls += u'\u2009%s' % hunit
    if ha == 'right':
        th = ax.text(x, 0.5*(y0+y1), ls, clip_on=False, rotation=90.0,
                     ha='left', va='center', **kwargs)
        # get x coordinate of text bottom in figure pixel coordinates:
        tx = np.array(th.get_window_extent(ax.get_figure().canvas.get_renderer()))[0,0]
        dtx = lx+0.5*lw + 2.0 - tx
    else:
        th = ax.text(x, 0.5*(y0+y1), ls, clip_on=False, rotation=90.0,
                     ha='right', va='center', **kwargs)
        # get x coordinate of text bottom in figure pixel coordinates:
        tx = np.array(th.get_window_extent(ax.get_figure().canvas.get_renderer()))[1,0]
        dtx = lx-0.5*lw - 1.0 - tx
    th.set_position((x+dxu*dtx, 0.5*(y0+y1)))
    return x, y0, y1

def arrowed_spines(ax, ms=10):
	""" Spine with arrow on the y-axis of a plot.

    Parameters
    ----------
    ax : matplotlib figure axis
        Axis on which the arrow should be plot.
	"""
	xmin, xmax = ax.get_xlim()
	ymin, ymax = ax.get_ylim()
	ax.scatter([xmin], [ymax], s=ms, marker='^', clip_on=False, color='k')
	ax.set_xlim(xmin, xmax)
	ax.set_ylim(ymin, ymax)

def loghist(ax, x, bmin, bmax, n, c, orientation='vertical', label=''):
	""" Plot histogram with logarithmic scale.

    Parameters
    ----------
    ax : matplotlib axis
        Axis to plot the histogram on.
    x : numpy array
        Input data for histogram.
    bmin : float
        Minimum value for the histogram bins.
    bmax : float
        Maximum value for the histogram bins.
    n : int
        Number of bins.
    c : matplotlib color
        Color of histogram.
    orientation : string (optional)
        Histogram orientation.
        Defaults to 'vertical'.
    label : string (optional)
        Label for x.
        Defaults to '' (no label).

    Returns
    -------
    n : array
        The values of the histogram bins.
    bins : array
        The edges of the bins.
    patches : BarContainer
        Container of individual artists used to create the histogram.
	"""
	return ax.hist(x, bins=np.exp(np.linspace(np.log(bmin), np.log(bmax), n)),
                   color=c, orientation=orientation, label=label)

def plot_all(data, eod_p_times, eod_tr_times, fs, mean_eods):
    """Quick way to view the output of extract_pulsefish in a single plot.

    Parameters
    ----------
    data: array
        Recording data.
    eod_p_times: array of ints
        EOD peak indices.
    eod_tr_times: array of ints
        EOD trough indices.
    fs: float
        Samplerate.
    mean_eods: list of numpy arrays
        Mean EODs of each pulsefish found in the recording.
    """
    fig = plt.figure(figsize=(10, 5))

    if len(eod_p_times) > 0:
        gs = gridspec.GridSpec(2, len(eod_p_times))
        ax = fig.add_subplot(gs[0,:])
        ax.plot(np.arange(len(data))/fs, data, c='k', alpha=0.3)

        for i, (pt, tt) in enumerate(zip(eod_p_times, eod_tr_times)):
            ax.plot(pt, data[(pt*fs).astype('int')], 'o', label=i+1, ms=10, c=cmap(i))
            ax.plot(tt, data[(tt*fs).astype('int')], 'o', label=i+1, ms=10, c=cmap(i))

        ax.set_xlabel('time [s]')
        ax.set_ylabel('amplitude [V]')

        for i, m in enumerate(mean_eods):
            ax = fig.add_subplot(gs[1,i])
            ax.plot(1000*m[0], 1000*m[1], c='k')

            ax.fill_between(1000*m[0], 1000*(m[1]-m[2]), 1000*(m[1]+m[2]), color=cmap(i))
            ax.set_xlabel('time [ms]')
            ax.set_ylabel('amplitude [mV]')
    else:
        plt.plot(np.arange(len(data))/fs, data, c='k', alpha=0.3)

    plt.tight_layout()

def plot_clustering(samplerate, eod_widths, eod_hights, eod_shapes, disc_masks, merge_masks):
	"""Plot all clustering steps.

	Plot clustering steps on width, height and shape. Then plot the remaining EODs after
	the EOD assessment step and the EODs after the merge step.

	Parameters
	----------
	samplerate : float
		Samplerate of EOD snippets.
	eod_widths : list of three 1D numpy arrays
		The first list entry gives the unique labels of all width clusters as a list of ints.
		The second list entry gives the width values for each EOD in samples as a
        1D numpy array of ints.
		The third list entry gives the width labels for each EOD as a 1D numpy array of ints.
	eod_hights : nested lists (2 layers) of three 1D numpy arrays
		The first list entry gives the unique labels of all height clusters as a list of ints
        for each width cluster.
		The second list entry gives the height values for each EOD as a 1D numpy array
        of floats for each width cluster.
		The third list entry gives the height labels for each EOD as a 1D numpy array
        of ints for each width cluster.
	eod_shapes : nested lists (3 layers) of three 1D numpy arrays
		The first list entry gives the raw EOD snippets as a 2D numpy array for each
        height cluster in a width cluster.
		The second list entry gives the snippet PCA values for each EOD as a 2D numpy array
        of floats for each height cluster in a width cluster.
		The third list entry gives the shape labels for each EOD as a 1D numpy array of ints
        for each height cluster in a width cluster.
	disc_masks : Nested lists (two layers) of 1D numpy arrays
		The masks of EODs that are discarded by the discarding step of the algorithm.
        The masks are 1D boolean arrays where
		instances that are set to True are discarded by the algorithm. Discarding masks
        are saved in nested lists that represent the width and height clusters.
	merge_masks : Nested lists (two layers) of 2D numpy arrays
		The masks of EODs that are discarded by the merging step of the algorithm.
        The masks are 2D boolean arrays where
		for each sample point `i` either `merge_mask[i,0]` or `merge_mask[i,1]` is set to True.
        Here, merge_mask[:,0] represents the
		peak-centered clusters and `merge_mask[:,1]` represents the trough-centered clusters.
        Merge masks are saved in nested lists
		that represent the width and height clusters.
	"""
	# create figure + transparant figure.
	fig = plt.figure(figsize=(12, 7))
	transFigure = fig.transFigure.inverted()

	# set up the figure layout
	outer = gridspec.GridSpec(1, 5, width_ratios=[1, 1, 2, 1, 2], left=0.05, right=0.95)

	# set titles for each clustering step
	titles = ['1. Widths', '2. Heights', '3. Shape', '4. Pulse EODs', '5. Merge']
	for i, title in enumerate(titles):
		title_ax = gridspec.GridSpecFromSubplotSpec(1, 1, subplot_spec = outer[i])
		ax = fig.add_subplot(title_ax[0])
		ax.text(0, 110, title, ha='center', va='bottom', clip_on=False)
		ax.set_xlim(-100, 100)
		ax.set_ylim(-100, 100)
		ax.axis('off')

	# compute sizes for each axis
	w_size = 1
	h_size = len(eod_hights[1])

	shape_size = np.sum([len(sl) for sl in eod_shapes[0]])

	# count required axes sized for the last two plot columns.
	disc_size = 0
	merge_size= 0
	for shapelabel, dmasks, mmasks in zip(eod_shapes[2], disc_masks, merge_masks):
		for sl, dm, mm in zip(shapelabel, dmasks, mmasks):
			uld1 = np.unique((sl[0]+1)*np.invert(dm[0]))
			uld2 = np.unique((sl[1]+1)*np.invert(dm[1]))
			disc_size = disc_size+len(uld1[uld1>0])+len(uld2[uld2>0])

			uld1 = np.unique((sl[0]+1)*mm[0])
			uld2 = np.unique((sl[1]+1)*mm[1])
			merge_size = merge_size+len(uld1[uld1>0])+len(uld2[uld2>0])

	# set counters to keep track of the plot axes
	disc_block = 0
	merge_block = 0
	shape_count = 0

	# create all axes
	width_hist_ax = gridspec.GridSpecFromSubplotSpec(w_size, 1, subplot_spec = outer[0])
	hight_hist_ax = gridspec.GridSpecFromSubplotSpec(h_size, 1, subplot_spec = outer[1])
	shape_ax = gridspec.GridSpecFromSubplotSpec(shape_size, 1,  subplot_spec = outer[2])
	shape_windows = [gridspec.GridSpecFromSubplotSpec(2, 2, hspace=0.0, wspace=0.0,
                                                      subplot_spec=shape_ax[i])
                        for i in range(shape_size)]

	EOD_delete_ax = gridspec.GridSpecFromSubplotSpec(disc_size, 1, subplot_spec=outer[3])
	EOD_merge_ax = gridspec.GridSpecFromSubplotSpec(merge_size, 1, subplot_spec=outer[4])

    # plot width labels histogram
	ax1 = fig.add_subplot(width_hist_ax[0])
	# set axes features.
	ax1.set_xscale('log')
	ax1.spines['top'].set_visible(False)
	ax1.spines['right'].set_visible(False)
	ax1.spines['bottom'].set_visible(False)
	ax1.axes.xaxis.set_visible(False)
	ax1.set_yticklabels([])

	# indices for plot colors (dark to light)
	colidxsw = -np.linspace(-1.25, -0.5, h_size)

	for i, (wl, colw, uhl, eod_h, eod_h_labs, w_snip, w_feat, w_lab, w_dm, w_mm) in enumerate(zip(eod_widths[0], colidxsw, eod_hights[0], eod_hights[1], eod_hights[2], eod_shapes[0], eod_shapes[1], eod_shapes[2], disc_masks, merge_masks)):

		# plot width hist
		hw, _, _ = ax1.hist(eod_widths[1][eod_widths[2]==wl],
                            bins=np.linspace(np.min(eod_widths[1]), np.max(eod_widths[1]), 100),
                            color=lighter(c_o, colw), orientation='horizontal')

		# set arrow when the last hist is plot so the size of the axes are known.
		if i == h_size-1:
			arrowed_spines(ax1, ms=20)

		# determine total size of the hight historgams now.
		my, b = np.histogram(eod_h, bins=np.exp(np.linspace(np.min(np.log(eod_h)),
                                                            np.max(np.log(eod_h)), 100)))
		maxy = np.max(my)

		# set axes features for hight hist.
		ax2 = fig.add_subplot(hight_hist_ax[h_size-i-1])
		ax2.set_xscale('log')
		ax2.spines['top'].set_visible(False)
		ax2.spines['right'].set_visible(False)
		ax2.spines['bottom'].set_visible(False)
		ax2.set_xlim(0.9, maxy)
		ax2.axes.xaxis.set_visible(False)
		ax2.set_yscale('log')
		ax2.yaxis.set_major_formatter(ticker.NullFormatter())
		ax2.yaxis.set_minor_formatter(ticker.NullFormatter())

		# define colors for plots
		colidxsh = -np.linspace(-1.25, -0.5, len(uhl))

		for n, (hl, hcol, snippets, features, labels, dmasks, mmasks) in enumerate(zip(uhl, colidxsh, w_snip, w_feat, w_lab, w_dm, w_mm)):

			hh, _, _ = loghist(ax2, eod_h[eod_h_labs==hl], np.min(eod_h), np.max(eod_h), 100,
                               lighter(c_g, hcol), orientation='horizontal')

			# set arrow spines only on last plot
			if n == len(uhl)-1:
				arrowed_spines(ax2, ms=10)

			# plot line from the width histogram to the height histogram.
			if n == 0:
				coord1 = transFigure.transform(ax1.transData.transform([np.median(hw[hw!=0]),
                                                                        np.median(eod_widths[1][eod_widths[2]==wl])]))
				coord2 = transFigure.transform(ax2.transData.transform([0.9, np.mean(eod_h)]))
				line = Line2D((coord1[0], coord2[0]), (coord1[1], coord2[1]),
                              transform=fig.transFigure, color='grey', linewidth=0.5)
				fig.lines.append(line)

			# compute sizes of the eod_discarding and merge steps
			s1 = np.unique((labels[0]+1)*(~dmasks[0]))
			s2 = np.unique((labels[1]+1)*(~dmasks[1]))
			disc_block = disc_block + len(s1[s1>0]) + len(s2[s2>0])

			s1 = np.unique((labels[0]+1)*(mmasks[0]))
			s2 = np.unique((labels[1]+1)*(mmasks[1]))
			merge_block = merge_block + len(s1[s1>0]) + len(s2[s2>0])

			axs = []
			disc_count = 0
			merge_count = 0

			# now plot the clusters for peak and trough centerings
			for pt, cmap_pt in zip([0, 1], cmap_pts):

				ax3 = fig.add_subplot(shape_windows[shape_size-1-shape_count][pt,0])
				ax4 = fig.add_subplot(shape_windows[shape_size-1-shape_count][pt,1])

				# remove axes
				ax3.axes.xaxis.set_visible(False)
				ax4.axes.yaxis.set_visible(False)
				ax3.axes.yaxis.set_visible(False)
				ax4.axes.xaxis.set_visible(False)

				# set color indices
				colidxss = -np.linspace(-1.25, -0.5, len(np.unique(labels[pt][labels[pt]>=0])))
				j=0
				for c in np.unique(labels[pt]):

					if c<0:
						# plot noise features + snippets
						ax3.plot(features[pt][labels[pt]==c,0], features[pt][labels[pt]==c,1],
                                 '.', color='lightgrey', label='-1', rasterized=True)
						ax4.plot(snippets[pt][labels[pt]==c].T, linewidth=0.1,
                                 color='lightgrey', label='-1', rasterized=True)
					else:
						# plot cluster features and snippets
						ax3.plot(features[pt][labels[pt]==c,0], features[pt][labels[pt]==c,1],
                                 '.', color=lighter(cmap_pt, colidxss[j]), label=c,
                                 rasterized=True)
						ax4.plot(snippets[pt][labels[pt]==c].T, linewidth=0.1,
                                 color=lighter(cmap_pt, colidxss[j]), label=c, rasterized=True)

						# check if the current cluster is an EOD, if yes, plot it.
						if np.sum(dmasks[pt][labels[pt]==c]) == 0:

							ax = fig.add_subplot(EOD_delete_ax[disc_size-disc_block+disc_count])
							ax.axis('off')

							# plot mean EOD snippet
							ax.plot(np.mean(snippets[pt][labels[pt]==c], axis=0),
                                    color=lighter(cmap_pt, colidxss[j]))
							disc_count = disc_count + 1

							# match colors and draw line..
							coord1 = transFigure.transform(ax4.transData.transform([ax4.get_xlim()[1],
                                                                                ax4.get_ylim()[0] + 0.5*(ax4.get_ylim()[1]-ax4.get_ylim()[0])]))
							coord2 = transFigure.transform(ax.transData.transform([ax.get_xlim()[0],ax.get_ylim()[0] + 0.5*(ax.get_ylim()[1]-ax.get_ylim()[0])]))
							line = Line2D((coord1[0], coord2[0]), (coord1[1], coord2[1]),
                                          transform=fig.transFigure, color='grey',
                                          linewidth=0.5)
							fig.lines.append(line)
							axs.append(ax)

							# check if the current EOD survives the merge step
							# if so, plot it.
							if np.sum(mmasks[pt, labels[pt]==c])>0:

								ax = fig.add_subplot(EOD_merge_ax[merge_size-merge_block+merge_count])
								ax.axis('off')

								ax.plot(np.mean(snippets[pt][labels[pt]==c], axis=0),
                                        color=lighter(cmap_pt, colidxss[j]))
								merge_count = merge_count + 1

						j=j+1

				if pt==0:
					# draw line from hight cluster to EOD shape clusters.
					coord1 = transFigure.transform(ax2.transData.transform([np.median(hh[hh!=0]),
                                                                            np.median(eod_h[eod_h_labs==hl])]))
					coord2 = transFigure.transform(ax3.transData.transform([ax3.get_xlim()[0],
                                                                            ax3.get_ylim()[0]]))
					line = Line2D((coord1[0], coord2[0]), (coord1[1], coord2[1]),
                                  transform=fig.transFigure, color='grey', linewidth=0.5)
					fig.lines.append(line)

			shape_count = shape_count + 1

			if len(axs)>0:
				# plot lines that indicate the merged clusters.
				coord1 = transFigure.transform(axs[0].transData.transform([axs[0].get_xlim()[1]+0.1*(axs[0].get_xlim()[1]-axs[0].get_xlim()[0]),
                                                                           axs[0].get_ylim()[1]-0.25*(axs[0].get_ylim()[1]-axs[0].get_ylim()[0])]))
				coord2 = transFigure.transform(axs[-1].transData.transform([axs[-1].get_xlim()[1]+0.1*(axs[-1].get_xlim()[1]-axs[-1].get_xlim()[0]),
                                                                            axs[-1].get_ylim()[0]+0.25*(axs[-1].get_ylim()[1]-axs[-1].get_ylim()[0])]))
				line = Line2D((coord1[0], coord2[0]), (coord1[1], coord2[1]),
                              transform=fig.transFigure, color='grey', linewidth=1)
				fig.lines.append(line)

def plot_bgm(x, means, variances, weights, use_log, labels, labels_am, xlab):
	"""Plot a BGM clustering step either on EOD width or height.

	Parameters
	----------
	x : 1D numpy array of floats
		BGM input values.
	means : list of floats
		BGM Gaussian means
	variances : list of floats
		BGM Gaussian variances.
	weights : list of floats
		BGM Gaussian weights.
	use_log : boolean
		True if the z-scored logarithm of the data was used as BGM input.
	labels : 1D numpy array of ints
		Labels defined by BGM model (before merging based on merge factor).
	labels_am : 1D numpy array of ints
		Labels defined by BGM model (after merging based on merge factor).
	xlab : string
		Label for plot (defines the units of the BGM data).
	"""
	if 'width' in xlab:
		ccol = c_o
	elif 'height' in xlab:
		ccol = c_g
	else:
		ccol = 'b'

	# get the transform that was used as BGM input
	if use_log:
		x_transform = stats.zscore(np.log(x))
		xplot = np.exp(np.linspace(np.log(np.min(x)), np.log(np.max(x)), 1000))
	else:
		x_transform = stats.zscore(x)
		xplot = np.linspace(np.min(x), np.max(x), 1000)

	# compute the x values and gaussians
	x2 = np.linspace(np.min(x_transform), np.max(x_transform), 1000)
	gaussians = []
	gmax = 0
	for i, (w, m, std) in enumerate(zip(weights, means, variances)):
		gaus = np.sqrt(w*stats.norm.pdf(x2, m, np.sqrt(std)))
		gaussians.append(gaus)
		gmax = max(np.max(gaus), gmax)

	# compute classes defined by gaussian intersections
	classes = np.argmax(np.vstack(gaussians), axis=0)

	# find the minimum of any gaussian that is within its class
	gmin = 100
	for i, c in enumerate(np.unique(classes)):
		gmin=min(gmin, np.min(gaussians[c][classes==c]))

	# set up the figure
	fig, ax1 = plt.subplots(figsize=(8, 4.8))
	fig_ysize = 4
	ax2 = ax1.twinx()
	ax1.spines['top'].set_visible(False)
	ax2.spines['top'].set_visible(False)
	ax1.set_xlabel('x [a.u.]')
	ax1.set_ylabel('#')
	ax2.set_ylabel('Likelihood')
	ax2.set_yscale('log')
	ax1.set_yscale('log')
	if use_log:
		ax1.set_xscale('log')
	ax1.set_xlabel(xlab)

	# define colors for plotting gaussians
	colidxs = -np.linspace(-1.25, -0.5, len(np.unique(classes)))

	# plot the gaussians
	for i, c in enumerate(np.unique(classes)):
		ax2.plot(xplot, gaussians[c], c=lighter(c_grey, colidxs[i]), linewidth=2,
                 label=r'$N(\mu_%i, \sigma_%i)$'%(c, c))

	# plot intersection lines
	ax2.vlines(xplot[1:][np.diff(classes)!=0], 0, gmax/gmin, color='k', linewidth=2,
               linestyle='--')
	ax2.set_ylim(gmin, np.max(np.vstack(gaussians))*1.1)

	# plot data distributions and classes
	colidxs = -np.linspace(-1.25, -0.5, len(np.unique(labels)))
	for i, l in enumerate(np.unique(labels)):
		if use_log:
			h, binn, _ = loghist(ax1, x[labels==l], np.min(x), np.max(x), 100,
                               lighter(ccol, colidxs[i]), label=r'$x_%i$'%l)
		else:
			h, binn, _ = ax1.hist(x[labels==l], bins=np.linspace(np.min(x), np.max(x), 100),
                                  color=lighter(ccol, colidxs[i]), label=r'$x_%i$'%l)

	# annotate merged clusters
	for l in np.unique(labels_am):
		maps = np.unique(labels[labels_am==l])
		if len(maps) > 1:
			x1 = x[labels==maps[0]]
			x2 = x[labels==maps[1]]

			print(np.median(x1))
			print(np.median(x2))
			print(gmax)
			ax2.plot([np.median(x1), np.median(x2)], [1.2*gmax, 1.2*gmax], c='k', clip_on=False)
			ax2.plot([np.median(x1), np.median(x1)], [1.1*gmax, 1.2*gmax], c='k', clip_on=False)
			ax2.plot([np.median(x2), np.median(x2)], [1.1*gmax, 1.2*gmax], c='k', clip_on=False)
			ax2.annotate(r'$\frac{|{\tilde{x}_%i-\tilde{x}_%i}|}{max(\tilde{x}_%i, \tilde{x}_%i)} < \epsilon$' % (maps[0], maps[1], maps[0], maps[1]), [np.median(x1)*1.1, gmax*1.2],  xytext=(10,  10),  textcoords='offset points', fontsize=12, annotation_clip=False, ha='center')

	# add legends and plot.
	ax2.legend(loc='lower left', frameon=False, bbox_to_anchor=(-0.05, 1.3),
               ncol=len(np.unique(classes)))
	ax1.legend(loc='upper left', frameon=False, bbox_to_anchor=(-0.05, 1.3),
               ncol=len(np.unique(labels)))
	plt.tight_layout()

def plot_feature_extraction(raw_snippets, normalized_snippets, features, labels, dt, pt):
	"""Plot clustering step on EOD shape.

	Parameters
	----------
	raw_snippets : 2D numpy array
		Raw EOD snippets.
	normalized_snippets : 2D numpy array
		Normalized EOD snippets.
	features : 2D numpy array
		PCA values for each normalized EOD snippet.
	labels : 1D numpy array of ints
		Cluster labels.
	dt : float
		Sample interval of snippets.
	pt : int
		Set to 0 for peak-centered EODs and set to 1 for trough-centered EODs.
	"""
	ccol = cmap_pts[pt]

	# set up the figure layout
	fig = plt.figure(figsize=(((2+0.2)*4.8), 4.8))
	outer = gridspec.GridSpec(1, 2, wspace=0.2, hspace=0)

	x = np.arange(-dt*1000*raw_snippets.shape[1]/2, dt*1000*raw_snippets.shape[1]/2, dt*1000)

	snip_ax = gridspec.GridSpecFromSubplotSpec(2, 1, subplot_spec = outer[0], hspace=0.35)
	pc_ax = gridspec.GridSpecFromSubplotSpec(features.shape[1]-1, features.shape[1]-1,
                                             subplot_spec = outer[1], hspace=0, wspace=0)

	# 3 plots: raw snippets, normalized, pcs.
	ax_raw_snip = fig.add_subplot(snip_ax[0])
	ax_normalized_snip = fig.add_subplot(snip_ax[1])

	colidxs = -np.linspace(-1.25, -0.5, len(np.unique(labels[labels>=0])))
	j=0

	for c in np.unique(labels):
		if c<0:
			color='lightgrey'
		else:
			color = lighter(ccol, colidxs[j])
			j=j+1

		ax_raw_snip.plot(x, raw_snippets[labels==c].T, color=color, label='-1',
                         rasterized=True, alpha=0.25)
		ax_normalized_snip.plot(x, normalized_snippets[labels==c].T, color=color, alpha=0.25)
		ax_raw_snip.spines['top'].set_visible(False)
		ax_raw_snip.spines['right'].set_visible(False)
		ax_raw_snip.get_xaxis().set_ticklabels([])
		ax_raw_snip.set_title('Raw snippets')
		ax_raw_snip.set_ylabel('Amplitude [a.u.]')
		ax_normalized_snip.spines['top'].set_visible(False)
		ax_normalized_snip.spines['right'].set_visible(False)
		ax_normalized_snip.set_title('Normalized snippets')
		ax_normalized_snip.set_ylabel('Amplitude [a.u.]')
		ax_normalized_snip.set_xlabel('Time [ms]')

		ax_raw_snip.axis('off')
		ax_normalized_snip.axis('off')

		ax_overlay = fig.add_subplot(pc_ax[:,:])
		ax_overlay.set_title('Features')
		ax_overlay.axis('off')

		for n in range(features.shape[1]):
			for m in range(n):
				ax = fig.add_subplot(pc_ax[n-1,m])
				ax.scatter(features[labels==c,m], features[labels==c,n], marker='.',
                           color=color, alpha=0.25)
				ax.set_xlim(np.min(features), np.max(features))
				ax.set_ylim(np.min(features), np.max(features))
				ax.get_xaxis().set_ticklabels([])
				ax.get_yaxis().set_ticklabels([])
				ax.get_xaxis().set_ticks([])
				ax.get_yaxis().set_ticks([])

				if m==0:
					ax.set_ylabel('PC %i'%(n+1))

				if n==features.shape[1]-1:
					ax.set_xlabel('PC %i'%(m+1))

		ax = fig.add_subplot(pc_ax[0,features.shape[1]-2])
		ax.set_xlim(np.min(features), np.max(features))
		ax.set_ylim(np.min(features), np.max(features))

		size = max(1, int(np.ceil(-np.log10(np.max(features)-np.min(features)))))
		wbar = np.floor((np.max(features)-np.min(features))*10**size)/10**size

		# should be smaller than the actual thing! so like x% of it?
		xscalebar(ax, 0, 0, wbar, wformat='%%.%if'%size)
		yscalebar(ax, 0, 0, wbar, hformat='%%.%if'%size)
		ax.axis('off')

def plot_moving_fish(ws, dts, clusterss, ts, fishcounts, T, ignore_stepss):
	"""Plot moving fish detection step.

	Parameters
	----------
	ws : list of floats
		Median width for each width cluster that the moving fish algorithm is computed on
        (in seconds).
	dts : list of floats
		Sliding window size (in seconds) for each width cluster.
	clusterss : list of 1D numpy int arrays
		Cluster labels for each EOD cluster in a width cluster.
	ts : list of 1D numpy float arrays
		EOD emission times for each EOD in a width cluster.
	fishcounts : list of lists
		Sliding window timepoints and fishcounts for each width cluster.
	T : float
		Lenght of analyzed recording in seconds.
	ignore_stepss : list of 1D int arrays
		Mask for fishcounts that were ignored (ignored if True) in the moving_fish analysis.
	"""
	fig = plt.figure()

	# create gridspec
	outer = gridspec.GridSpec(len(ws), 1)

	for i, (w, dt, clusters, t, fishcount, ignore_steps) in enumerate(zip(ws, dts, clusterss, ts, fishcounts, ignore_stepss)):

		gs = gridspec.GridSpecFromSubplotSpec(2, 1, subplot_spec = outer[i])

		# axis for clusters
		ax1 = fig.add_subplot(gs[0])
		# axis for fishcount
		ax2 = fig.add_subplot(gs[1])

		# plot clusters as eventplot
		for cnum, c in enumerate(np.unique(clusters[clusters>=0])):
			ax1.eventplot(t[clusters==c], lineoffsets=cnum, linelengths=0.5, color=cmap(i))
			cnum = cnum + 1

		# Plot the sliding window
		rect=Rectangle((0, -0.5), dt, cnum, linewidth=1, linestyle='--', edgecolor='k',
                       facecolor='none', clip_on=False)
		ax1.add_patch(rect)
		ax1.arrow(dt+0.1, -0.5,  0.5, 0, head_width=0.1, head_length=0.1, facecolor='k',
                  edgecolor='k')

		# plot parameters
		ax1.set_title(r'$\tilde{w}_%i = %.3f ms$'%(i, 1000*w))
		ax1.set_ylabel('cluster #')
		ax1.set_yticks(range(0, cnum))
		ax1.set_xlabel('time')
		ax1.set_xlim(0, T)
		ax1.axes.xaxis.set_visible(False)
		ax1.spines['bottom'].set_visible(False)
		ax1.spines['top'].set_visible(False)
		ax1.spines['right'].set_visible(False)
		ax1.spines['left'].set_visible(False)

		# plot for fishcount
		x = fishcount[0]
		y = fishcount[1]

		ax2 = fig.add_subplot(gs[1])
		ax2.spines['top'].set_visible(False)
		ax2.spines['right'].set_visible(False)
		ax2.spines['bottom'].set_visible(False)
		ax2.axes.xaxis.set_visible(False)

		yplot = np.copy(y)
		ax2.plot(x+dt/2, yplot, linestyle='-', marker='.', c=cmap(i), alpha=0.25)
		yplot[ignore_steps.astype(bool)] = np.NaN
		ax2.plot(x+dt/2, yplot, linestyle='-', marker='.', c=cmap(i))
		ax2.set_ylabel('Fish count')
		ax2.set_yticks(range(int(np.min(y)), 1+int(np.max(y))))
		ax2.set_xlim(0, T)

		if i < len(ws)-1:
		    ax2.axes.xaxis.set_visible(False)
		else:
			ax2.axes.xaxis.set_visible(False)
			xscalebar(ax2, 1, 0, 1, wunit='s', ha='right')

		con = ConnectionPatch([0, -0.5],  [dt/2, y[0]], "data", "data",
		    axesA=ax1, axesB=ax2, color='k')
		ax2.add_artist(con)
		con = ConnectionPatch([dt, -0.5], [dt/2, y[0]], "data", "data",
		    axesA=ax1, axesB=ax2, color='k')
		ax2.add_artist(con)

		plt.xlim(0, T)


import numpy as np
import matplotlib.pyplot as plt
import time
from BZI.symmetry import make_ptvecs
from BZI.sampling import make_grid
from BZI.pseudopots import Al_PP
from BZI.integration import monte_carlo
from BZI.plots import PlotMesh

class Convergence(object):
    """ Compare integrations of pseudo-potentials by creating convergence plots.

    Args:
        pseudo_potential (function): a pseudo-potential function taken from
            BZI.pseudopots
        cutoff (float): the energy cutoff of the pseudo-potential
        cell_type (str): the geometry of the integration cell
        cell_constant (float): the size of the integration cell
        offset (list): a vector that offsets the grid from the origin and is
            given in grid coordinates.
        grid_types (list): a list of grid types
        grid_constants (list): a list of grid constants
        integration_methods (list): a list of integration methods

    Attributes:
        pseudo_potential (function): a pseudo-potential function taken from
            BZI.pseudopots
        cell_type (str): the geometry of the integration cell.
        cell_constant (float): the size of the integration cell.
        cell_vectors (np.ndarray): an array vectors as columns of a 3x3 numpy
            array that is used to create the cell
        grid_types (list): a list of grid types
        grid_constants (list): a list of grid constants
        integration_methods (list): a list of integration methods
        answer (float): the expected result of integration
        errors (list): a list of errors for each grid type
        nspts (list): a list of the number of sampling points for each grid type
        integrals (list): a list of integral value for each grid type and constant
        times (list): a list of the amount of time taken computing the grid
            generation and integration.
    """

    def __init__(self, pseudo_potential=None, cutoff=None, cell_centering=None,
                 cell_constants=None, cell_angles=None, offset=None,
                 grid_types=None, grid_constants=None,
                 integration_methods=None, origin=None, random = None):
        self.pseudo_potential = pseudo_potential or Al_PP
        self.cutoff = cutoff or 4.
        self.cell_centering = cell_centering or "prim"
        self.cell_constants = cell_constants or [1.]*3
        self.cell_angles = cell_angles or [np.pi/2]*3
        self.cell_vectors = make_ptvecs(self.cell_centering, self.cell_constants,
                                        self.cell_angles)
        self.grid_centerings = grid_centerings or ["prim", "base", "body", "face"]
        self.grid_constants = grid_constants or [1/n for n in range(2,11)]
        self.offset = offset or [0.,0.,0.]
        # self.integration_methods = integration_methods or [rectangle_method]
        self.origin = origin or [0.,0.,0.]
        self.random = random or False

    def compare_grids(self, answer, plot=False, save=False):
        self.answer = answer
        if self.random:
            nm = len(self.grid_types)
            self.nspts = [[] for _ in range(nm + 1)]
            self.errors = [[] for _ in range(nm + 1)]
            self.integrals = [[] for _ in range(nm + 1)]
            self.times = [[] for _ in range(nm + 1)]

            npts_list = [2**n for n in range(8,14)]
            for npts in npts_list:
                time1 = time.time()
                integral = monte_carlo(self.pseudo_potential,
                                       self.cell_vectors,
                                       npts,
                                       self.cutoff)
                self.nspts[nm].append(npts)
                self.integrals[nm].append(integral)
                self.times[nm].append((time.time() - time1))
                self.errors[nm].append(np.abs(self.integrals[nm][-1] - answer))
        else:
            self.nspts = [[] for _ in range(len(self.grid_types))]
            self.errors = [[] for _ in range(len(self.grid_types))]
            self.integrals = [[] for _ in range(len(self.grid_types))]
            self.times = [[] for _ in range(len(self.grid_types))]
        integration_method = self.integration_methods[0]
        for (i,grid_centering) in enumerate(self.grid_centering_list):
            for grid_consts in self.grid_constants_list:
                for grid_angles in grid_angles_list:
                    grid_vecs = make_ptvecs(grid_centering, grid_consts, grid_angles)
                    time1 = time.time()
                    npts, integral = integration_method(self.pseudo_potential,
                                                                self.cell_vectors,
                                                                grid_vecs,
                                                                self.offset,
                                                                self.origin,
                                                                self.cutoff)
                self.nspts[i].append(npts)
                self.integrals[i].append(integral)
                self.times[i].append((time.time() - time1))
                self.errors[i].append(np.abs(self.integrals[i][-1] - answer))

        if save:
            np.save("%s_times" %self.pseudo_potential, self.times)
            np.save("%s_integrals" %self.pseudo_potential, self.integrals)
            np.save("%s_errors" %self.pseudo_potential, self.errors)

        if plot:
            if self.random:
                plt.loglog(self.nspts[nm], self.errors[nm], label="random", color="orange")
            for i in range(len(self.grid_types)):
                plt.loglog(self.nspts[i], self.errors[i], label=self.grid_types[i])
            plt.xlabel("Number of samping points")
            plt.ylabel("Error")
            test = [1./n**(2./3) for n in self.nspts[0]]
            plt.loglog(self.nspts[0], test, label="1/n**(2/3)")
            lgd = plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
            plt.grid()
            plt.show()
            plt.close()
            for i in range(len(self.grid_types)):
                plt.loglog(self.nspts[i], self.times[i], label=self.grid_types[i])
            plt.xlabel("Number of samping points")
            plt.ylabel("Time (s)")
            lgd = plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
            plt.grid()
            plt.show()
            plt.close()

    def plot_grid(self,i,j):
        """Plot one of the grids in the convergence plot.
        """
        grid_vecs = make_ptvecs(self.grid_types[i], self.grid_constants[j])
        grid_pts = make_grid(self.rcell_vectors, gr_vecs, self.offset)

        PlotMesh(grid_pts, self.rcell_vectors, self.offset)


"""
===================================
Swiss Roll reduction with LLE
===================================

An illustration of Swiss Roll reduction
with locally linear embedding
"""

# Author: Fabian Pedregosa -- <fabian.pedregosa@inria.fr>
# License: BSD 3 clause (C) INRIA 2011

print(__doc__)

import matplotlib.pyplot as plt

# This import is needed to modify the way figure behaves
from mpl_toolkits.mplot3d import Axes3D
Axes3D

#----------------------------------------------------------------------
# Locally linear embedding of the swiss roll

from sklearn import manifold, datasets
X, color = datasets.samples_generator.make_swiss_roll(n_samples=1500)

print("Computing LLE embedding")
X_r, err = manifold.locally_linear_embedding(X, n_neighbors=12,
                                             n_components=2)
print("Done. Reconstruction error: %g" % err)

#----------------------------------------------------------------------
# Plot result

fig = plt.figure()

ax = fig.add_subplot(211, projection='3d')
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral)

ax.set_title("Original data")
ax = fig.add_subplot(212)
ax.scatter(X_r[:, 0], X_r[:, 1], c=color, cmap=plt.cm.Spectral)
plt.axis('tight')
plt.xticks([]), plt.yticks([])
plt.title('Projected data')
plt.show()


import matplotlib.pyplot as plt

#
# plotting
#
def plot_elevation(ds, ax=None):
    if ax is None:
        ax = plt.gca()
    ds['hs'].plot(ax=ax,label="surface")
    ds['hb'].plot(ax=ax,label="bottom")

    # add horizontal line to indicate sea level
    ax.hlines(0, ds.x[0], ds.x[-1], linestyle='dashed', color='black')
    ds['zb'].plot(ax=ax, color='black', linewidth=2, label="bedrock") # add bedrock
    ax.legend(frameon=False, loc="upper right")

def plot_velocity(ds, ax=None):
    if ax is None:
        ax = plt.gca()
    ds = ds.copy()
    u = 'u' if 'u' in ds else 'U'
    ds[u] = ds[u]*3600*24
    ds[u].plot(ax=ax)
    ax.set_ylabel('velocity [m/d]')

def plot_glacier(ds):
    fig,axes=plt.subplots(2,1,sharex=True)
    ax=axes[0]
    plot_elevation(ds, ax)
    ax=axes[1]
    plot_velocity(ds, ax)
    ax.set_xlim([ds.x[0], ds.x[-1]])
    return fig, axes

def plot_stress(ds):
    _v = ["driving", "lat", "long", "basal", "residual"]
    try:
        ds = ds.take(_v)
    except KeyError:
        ds = ds.take([k + '_stress' for k in _v])
    return ds.to_array(axis='stress').T.plot()


from __future__ import print_function
import msgpack
import sys
import os
from collections import defaultdict
from helpers import text_dict_to_term_dict
from WikiExtractor import clean, compact
import pandas as pd

def remove_wikipedia_markup(text):
    return compact(clean(text.decode('utf8')))

def print_help_and_exit(msg=''):
    if msg:
        print('Error: {}\n'.format(msg))

    print('Usage: python make_tf_differences.py [n-grams] [path to directory]')
    print('The directory should contain files output by grab_texts.py')
    sys.exit(1)

if len(sys.argv) <= 2:
    print_help_and_exit()

##############################################################
# Read in msgpack files, separating them from simple and en Wikipedia
##############################################################
ngrams = int(sys.argv[1])
text_dir = sys.argv[2]

only = sys.argv[3:]
print('Only calculating for: ', only)
try:
    files = os.listdir(text_dir)
except OSError:
    print_help_and_exit()

##############################################################
# Organize the text files by subject, then wiki (en or simple)
##############################################################
file_dict = defaultdict(dict)
for f in files:
    try:
        subject, wiki, _ = f.split('_')
        if only and subject not in only:
            continue
        file_dict[subject][wiki] = f
    except ValueError:
        print_help_and_exit('Text directory does not contain valid filenames')

for subject in file_dict:
    print('Importing ', subject)
    with open(os.path.join(text_dir, file_dict[subject]['en'])) as f:
        en_text = msgpack.load(f)
        en_text = {k: remove_wikipedia_markup(v) for k,v in en_text.items()}

    with open(os.path.join(text_dir, file_dict[subject]['simple'])) as f:
        sm_text = msgpack.load(f)
        sm_text = {k: remove_wikipedia_markup(v) for k,v in sm_text.items()}

    print('Calculating term differences')
    en_tf, en_counts = text_dict_to_term_dict(en_text, ngrams)
    sm_tf, sm_counts = text_dict_to_term_dict(sm_text, ngrams)

    sm_terms = set(sm_tf)
    en_terms = set(en_tf)
    term_differences = {}

    for t in sm_terms.union(en_terms):
        term_differences[t] = en_tf[t] - sm_tf[t]

    sorted_term_difference = sorted(term_differences.items(),
            key=lambda x: x[1])

    print('Outputting term differences')
    td_df = pd.DataFrame(sorted_term_difference, columns=['term',
        'term_difference'])

    td_df['en_tf'] = td_df.term.apply(lambda x: en_tf[x])
    td_df['sm_tf'] = td_df.term.apply(lambda x: sm_tf[x])

    try:
        os.mkdir('data/term-diffs/ngrams-{}'.format(ngrams))
    except OSError:
        pass

    td_df.to_csv('data/term-diffs/ngrams-{}/{}_td.csv'.format(ngrams, subject),
            index=False, encoding='utf8')


"""
============
Mean filters
============

This example compares the following mean filters of the rank filter package:

* **local mean**: all pixels belonging to the structuring element to compute
  average gray level.
* **percentile mean**: only use values between percentiles p0 and p1
  (here 10% and 90%).
* **bilateral mean**: only use pixels of the structuring element having a gray
  level situated inside g-s0 and g+s1 (here g-500 and g+500)

Percentile and usual mean give here similar results, these filters smooth the
complete image (background and details). Bilateral mean exhibits a high
filtering rate for continuous area (i.e. background) while higher image
frequencies remain untouched.

"""
import numpy as np
import matplotlib.pyplot as plt

from skimage import data
from skimage.morphology import disk
from skimage.filters import rank

image = data.coins()
selem = disk(20)

percentile_result = rank.mean_percentile(image, selem=selem, p0=.1, p1=.9)
bilateral_result = rank.mean_bilateral(image, selem=selem, s0=500, s1=500)
normal_result = rank.mean(image, selem=selem)

fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(8, 10),
                         sharex=True, sharey=True)
ax = axes.ravel()

titles = ['Original', 'Percentile mean', 'Bilateral mean', 'Local mean']
imgs = [image, percentile_result, bilateral_result, normal_result]
for n in range(0, len(imgs)):
    ax[n].imshow(imgs[n])
    ax[n].set_title(titles[n])
    ax[n].set_adjustable('box-forced')
    ax[n].axis('off')

plt.show()


# -*- coding: utf-8 -*-
"""

Vincent Map Examples

"""

#Build a map from scratch

from vincent import *

world_topo = r'world-countries.topo.json'
state_topo = r'us_states.topo.json'
lake_topo = r'lakes_50m.topo.json'
county_geo = r'us_counties.geo.json'
county_topo = r'us_counties.topo.json'
or_topo = r'or_counties.topo.json'

vis = Visualization(width=960, height=500)
vis.data['countries'] = Data(
    name='countries',
    url=world_topo,
    format={'type': 'topojson', 'feature': 'world-countries'}
    )

geo_transform = Transform(
                type='geopath', value="data", projection='winkel3', scale=200,
                translate=[480, 250]
                )

geo_from = MarkRef(data='countries', transform=[geo_transform])

enter_props = PropertySet(
    stroke=ValueRef(value='#000000'),
    path=ValueRef(field='path')
    )

update_props = PropertySet(fill=ValueRef(value='steelblue'))

mark_props = MarkProperties(enter=enter_props, update=update_props)

vis.marks.append(
    Mark(type='path', from_=geo_from, properties=mark_props)
    )

vis.to_json('vega.json')

#Convenience Method

geo_data = [{'name': 'countries',
             'url': world_topo,
             'feature': 'world-countries'}]

vis = Map(geo_data=geo_data, scale=200)
vis.to_json('vega.json')

#States & Counties

geo_data = [{'name': 'counties',
             'url': county_topo,
             'feature': 'us_counties.geo'},
            {'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'}
             ]

vis = Map(geo_data=geo_data, scale=1000, projection='albersUsa')
del vis.marks[1].properties.update
vis.marks[0].properties.update.fill.value = '#084081'
vis.marks[1].properties.enter.stroke.value = '#fff'
vis.marks[0].properties.enter.stroke.value = '#7bccc4'
vis.to_json('vega.json')

#Choropleth
import json
import pandas as pd
#Map the county codes we have in our geometry to those in the
#county_data file, which contains additional rows we don't need
with open('us_counties.topo.json', 'r') as f:
    get_id = json.load(f)

#A little FIPS code munging
new_geoms = []
for geom in get_id['objects']['us_counties.geo']['geometries']:
    geom['properties']['FIPS'] = int(geom['properties']['FIPS'])
    new_geoms.append(geom)

get_id['objects']['us_counties.geo']['geometries'] = new_geoms

with open('us_counties.topo.json', 'w') as f:
    json.dump(get_id, f)

#Grab the FIPS codes and load them into a dataframe
geometries = get_id['objects']['us_counties.geo']['geometries']
county_codes = [x['properties']['FIPS'] for x in geometries]
county_df = pd.DataFrame({'FIPS': county_codes}, dtype=str)
county_df = county_df.astype(int)

#Read into Dataframe, cast to int for consistency
df = pd.read_csv('data/us_county_data.csv', na_values=[' '])
df['FIPS'] = df['FIPS'].astype(int)

#Perform an inner join, pad NA's with data from nearest county
merged = pd.merge(df, county_df, on='FIPS', how='inner')
merged = merged.fillna(method='pad')

geo_data = [{'name': 'counties',
             'url': county_topo,
             'feature': 'us_counties.geo'}]

vis = Map(data=merged, geo_data=geo_data, scale=1100, projection='albersUsa',
          data_bind='Employed_2011', data_key='FIPS',
          map_key={'counties': 'properties.FIPS'})
vis.marks[0].properties.enter.stroke_opacity = ValueRef(value=0.5)
#Change our domain for an even inteager
vis.scales['color'].domain = [0, 189000]
vis.legend(title='Number Employed 2011')
vis.to_json('vega.json')

#Lets look at different stats
vis.rebind(column='Civilian_labor_force_2011', brew='BuPu')
vis.to_json('vega.json')

vis.rebind(column='Unemployed_2011', brew='PuBu')
vis.to_json('vega.json')

vis.rebind(column='Unemployment_rate_2011', brew='YlGnBu')
vis.to_json('vega.json')

vis.rebind(column='Median_Household_Income_2011', brew='RdPu')
vis.to_json('vega.json')

#Mapping US State Level Data

state_data = pd.read_csv('data/US_Unemployment_Oct2012.csv')
geo_data = [{'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'}]
vis = Map(data=state_data, geo_data=geo_data, scale=1000,
          projection='albersUsa', data_bind='Unemployment', data_key='NAME',
          map_key={'states': 'properties.NAME'})
vis.legend(title='Unemployment (%)')
vis.to_json('vega.json')

#Iterating State Level Data
yoy = pd.read_table('data/State_Unemp_YoY.txt', delim_whitespace=True)
#Standardize State names to match TopoJSON for keying
names = []
for row in yoy.iterrows():
    pieces = row[1]['NAME'].split('_')
    together = ' '.join(pieces)
    names.append(together.title())
yoy['NAME'] = names
geo_data = [{'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'}]
vis = Map(data=yoy, geo_data=geo_data, scale=1000,
          projection='albersUsa', data_bind='AUG_2012', data_key='NAME',
          map_key={'states': 'properties.NAME'}, brew='YlGnBu')
#Custom threshold scale
vis.scales[0].type='threshold'
vis.scales[0].domain = [0, 2, 4, 6, 8, 10, 12]
vis.legend(title='Unemployment (%)')
vis.to_json('vega.json')

#Rebind and set our scale again
vis.rebind(column='AUG_2013', brew='YlGnBu')
vis.scales[0].type='threshold'
vis.scales[0].domain = [0, 2, 4, 6, 8, 10, 12]
vis.to_json('vega.json')

vis.rebind(column='CHANGE', brew='YlGnBu')
vis.scales[0].type='threshold'
vis.scales[0].domain = [-1.5, -1.3, -1.1, 0, 0.1, 0.3, 0.5, 0.8]
vis.legends[0].title = "YoY Change in Unemployment (%)"
vis.to_json('vega.json')

#Oregon County-level population data
or_data = pd.read_table('data/OR_County_Data.txt', delim_whitespace=True)
or_data['July_2012_Pop']= or_data['July_2012_Pop'].astype(int)
#Standardize keys
with open('or_counties.topo.json', 'r') as f:
    counties = json.load(f)

def split_county(name):
    parts = name.split(' ')
    parts.pop(-1)
    return ''.join(parts).upper()

#A little FIPS code munging
new_geoms = []
for geom in counties['objects']['or_counties.geo']['geometries']:
    geom['properties']['COUNTY'] = split_county(geom['properties']['COUNTY'])
    new_geoms.append(geom)

counties['objects']['or_counties.geo']['geometries'] = new_geoms

with open('or_counties.topo.json', 'w') as f:
    json.dump(counties, f)

geo_data = [{'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'},
            {'name': 'or_counties',
             'url': or_topo,
             'feature': 'or_counties.geo'}]

vis = Map(data=or_data, geo_data=geo_data, scale=3700,
          translate=[1480, 830],
          projection='albersUsa', data_bind='July_2012_Pop', data_key='NAME',
          map_key={'or_counties': 'properties.COUNTY'})
vis.marks[0].properties.update.fill.value = '#c2c2c2'
vis.to_json('vega.json')


"""
======================
SVM with custom kernel
======================

Simple usage of Support Vector Machines to classify a sample. It will
plot the decision surface and the support vectors.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
Y = iris.target

def my_kernel(X, Y):
    """
    We create a custom kernel:

                 (2  0)
    k(X, Y) = X  (    ) Y.T
                 (0  1)
    """
    M = np.array([[2, 0], [0, 1.0]])
    return np.dot(np.dot(X, M), Y.T)

h = .02  # step size in the mesh

# we create an instance of SVM and fit out data.
clf = svm.SVC(kernel=my_kernel)
clf.fit(X, Y)

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, m_max]x[y_min, y_max].
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)

# Plot also the training points
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)
plt.title('3-Class classification using Support Vector Machine with custom'
          ' kernel')
plt.axis('tight')
plt.show()


#
# Title  : Data munging(AKA cleaning) the Titanic Data
# Author : Felan Carlo Garcia
#
# Notes:
#  -- Code is based on the Kaggle Python Tutorial
#  -- data cleaning prior to implementing a machine learning algorithm.

import numpy as np
import pandas as pd

def processdata(filename, outputname):

  df = pd.read_csv(filename,header=0)

  # Make a new column 'Gender' and EmbarkedNum to convert the string
  # information into an integer value.
  # We do this because general machine learning algorithms do not
  # work on string values.
  df['Gender']      = df['Sex'].map({'female': 0, 'male': 1}).astype(int)
  df['EmbarkedNum'] = df['Embarked'].map({'S': 0, 'C': 1, 'Q': 1}).astype(int)

  # Executing the code:
  # --print df[df['Age'].isnull()][Sex']-- shows that the titanic data contains
  # some null values of the ages of the passengers.
  # In this case, we can either drop the row or we can assign an arbitrary
  # value to fill the missing data.
  # For this code, arbitrary age data is obtained by using the median
  # age data of the passengers. We make a new column 'AgeFill' and place
  # the median data on the missing values instead of directly modifying
  # the 'Age' column

  df['AgeFill'] = df['Age']
  for i in range(0, 2):
    for j in range(0, 3):
      median = df[(df['Gender'] == i) & (df['Pclass'] == j+1)]['Age'].dropna().median()
      df.loc[ (df.Age.isnull()) & (df.Gender == i) & (df.Pclass == j+1),'AgeFill'] = median

  # We add a new column 'AgeIsNull' to know which records has a missing
  # values previously.
  # We then interpolate the missing values from the 'Fare' column.
  df['AgeIsNull'] = pd.isnull(df.Age).astype(int)
  df['Fare']      = df['Fare'].interpolate()

  # ------------- Feature Engineering Part --------------------
  # Feature Engineering is the process of using domain/expert
  # knowledge of the data to create features that make machine
  # learning algorithms work better.
  #
  # In this case, studying the data shows that women and children
  # have higher survival rates compared to men. Thus we add
  # two additional features: 'Female' and 'Children', in an attempt
  # to assist our learning model in its prediction.
  # At the same time we add features Age*Class and FamilySize
  # as additional engineered feature that may help our learning
  # model
  df['Children']   = df['AgeFill'].map(lambda x: 1 if x < 6.0 else 0)
  df['Female']     = df['Gender'].map(lambda x: 1 if x == 0 else 0)
  df['FamilySize'] = df['SibSp']   + df['Parch']
  df['Age*Class']  = df['AgeFill'] * df['Pclass']

  # Since most machine learning algorithms don't work on strings,
  # we drop the columns in our pandas dataframe containing object
  # datatypes.
  # The code:
  # --print df.dtypes[df.dtypes.map(lambda x: x=='object')]--
  # will show which columns are made of object datatypes.
  #
  # In this case these are the following columns containing
  # object.string:
  # Age, Name, Sex, Ticket, Cabin, Embarked, Fare
  #
  # We drop the following objects columns along with the other data
  # since they wont likely contribute to our machine learning
  # prediction
  df = df.drop(['Age','Name', 'Sex', 'Ticket', 'Cabin', 'Embarked'], axis=1)
  df.to_csv(outputname, sep=',', index=False)

  return df

def main():
  print processdata('titanic-data-shuffled.csv', 'final-data.csv')

if __name__ == '__main__':
  main()


"""Plot latency data from messaging benchmarks.

To generate the data for each library, I started the server and then did
the following for each client::

    from xmlrpc_client import client
    for i in range(9):
        s = '0'*10**i
        print s
        %timeit client.echo(s)
"""

from matplotlib.pylab import *

rawdata = """# Data in milliseconds
Bytes	JSONRPC	PYRO	XMLRPC	pyzmq_copy	pyzmq_nocopy
1	2.15	0.186	2.07	0.111	0.136
10	2.49	0.187	1.87	0.115	0.137
100	2.5	0.189	1.9	0.126	0.138
1000	2.54	0.196	1.91	0.129	0.141
10000	2.91	0.271	2.77	0.204	0.197
100000	6.65	1.44	9.17	0.961	0.546
1000000	50.2	15.8	81.5	8.39	2.25
10000000	491	159	816	91.7	25.2
100000000	5010	1560	8300	893	248

"""
with open('latency.csv','w') as f:
    f.writelines(rawdata)

data = csv2rec('latency.csv',delimiter='\t')

loglog(data.bytes, data.xmlrpc*1000, label='XMLRPC')
loglog(data.bytes, data.jsonrpc*1000, label='JSONRPC')
loglog(data.bytes, data.pyro*1000, label='Pyro')
loglog(data.bytes, data.pyzmq_nocopy*1000, label='PyZMQ')
loglog(data.bytes, len(data.bytes)*[60], label='Ping')
legend(loc=2)
title('Latency')
xlabel('Number of bytes')
ylabel('Round trip latency ($\mu s$)')
grid(True)
show()
savefig('latency.png')

clf()

semilogx(data.bytes, 1000/data.xmlrpc, label='XMLRPC')
semilogx(data.bytes, 1000/data.jsonrpc, label='JSONRPC')
semilogx(data.bytes, 1000/data.pyro, label='Pyro')
semilogx(data.bytes, 1000/data.pyzmq_nocopy, label='PyZMQ')
legend(loc=1)
xlabel('Number of bytes')
ylabel('Message/s')
title('Message Throughput')
grid(True)
show()
savefig('msgs_sec.png')

clf()

loglog(data.bytes, 1000/data.xmlrpc, label='XMLRPC')
loglog(data.bytes, 1000/data.jsonrpc, label='JSONRPC')
loglog(data.bytes, 1000/data.pyro, label='Pyro')
loglog(data.bytes, 1000/data.pyzmq_nocopy, label='PyZMQ')
legend(loc=3)
xlabel('Number of bytes')
ylabel('Message/s')
title('Message Throughput')
grid(True)
show()
savefig('msgs_sec_log.png')

clf()

semilogx(data.bytes, data.pyro/data.pyzmq_nocopy, label="No-copy")
semilogx(data.bytes, data.pyro/data.pyzmq_copy, label="Copy")
xlabel('Number of bytes')
ylabel('Ratio throughputs')
title('PyZMQ Throughput/Pyro Throughput')
grid(True)
legend(loc=2)
show()
savefig('msgs_sec_ratio.png')


from datetime import datetime
import warnings

import numpy as np
import pytest

from pandas.core.dtypes.generic import ABCDateOffset

import pandas as pd
from pandas import (
    DatetimeIndex,
    Index,
    PeriodIndex,
    Series,
    Timestamp,
    bdate_range,
    date_range,
)
from pandas.tests.test_base import Ops
import pandas.util.testing as tm

from pandas.tseries.offsets import BDay, BMonthEnd, CDay, Day, Hour

START, END = datetime(2009, 1, 1), datetime(2010, 1, 1)

class TestDatetimeIndexOps(Ops):
    def setup_method(self, method):
        super().setup_method(method)
        mask = lambda x: (isinstance(x, DatetimeIndex) or isinstance(x, PeriodIndex))
        self.is_valid_objs = [o for o in self.objs if mask(o)]
        self.not_valid_objs = [o for o in self.objs if not mask(o)]

    def test_ops_properties(self):
        f = lambda x: isinstance(x, DatetimeIndex)
        self.check_ops_properties(DatetimeIndex._field_ops, f)
        self.check_ops_properties(DatetimeIndex._object_ops, f)
        self.check_ops_properties(DatetimeIndex._bool_ops, f)

    def test_ops_properties_basic(self):

        # sanity check that the behavior didn't change
        # GH#7206
        msg = "'Series' object has no attribute '{}'"
        for op in ["year", "day", "second", "weekday"]:
            with pytest.raises(AttributeError, match=msg.format(op)):
                getattr(self.dt_series, op)

        # attribute access should still work!
        s = Series(dict(year=2000, month=1, day=10))
        assert s.year == 2000
        assert s.month == 1
        assert s.day == 10
        msg = "'Series' object has no attribute 'weekday'"
        with pytest.raises(AttributeError, match=msg):
            s.weekday

    def test_repeat_range(self, tz_naive_fixture):
        tz = tz_naive_fixture
        rng = date_range("1/1/2000", "1/1/2001")

        result = rng.repeat(5)
        assert result.freq is None
        assert len(result) == 5 * len(rng)

        index = pd.date_range("2001-01-01", periods=2, freq="D", tz=tz)
        exp = pd.DatetimeIndex(
            ["2001-01-01", "2001-01-01", "2001-01-02", "2001-01-02"], tz=tz
        )
        for res in [index.repeat(2), np.repeat(index, 2)]:
            tm.assert_index_equal(res, exp)
            assert res.freq is None

        index = pd.date_range("2001-01-01", periods=2, freq="2D", tz=tz)
        exp = pd.DatetimeIndex(
            ["2001-01-01", "2001-01-01", "2001-01-03", "2001-01-03"], tz=tz
        )
        for res in [index.repeat(2), np.repeat(index, 2)]:
            tm.assert_index_equal(res, exp)
            assert res.freq is None

        index = pd.DatetimeIndex(["2001-01-01", "NaT", "2003-01-01"], tz=tz)
        exp = pd.DatetimeIndex(
            [
                "2001-01-01",
                "2001-01-01",
                "2001-01-01",
                "NaT",
                "NaT",
                "NaT",
                "2003-01-01",
                "2003-01-01",
                "2003-01-01",
            ],
            tz=tz,
        )
        for res in [index.repeat(3), np.repeat(index, 3)]:
            tm.assert_index_equal(res, exp)
            assert res.freq is None

    def test_repeat(self, tz_naive_fixture):
        tz = tz_naive_fixture
        reps = 2
        msg = "the 'axis' parameter is not supported"

        rng = pd.date_range(start="2016-01-01", periods=2, freq="30Min", tz=tz)

        expected_rng = DatetimeIndex(
            [
                Timestamp("2016-01-01 00:00:00", tz=tz, freq="30T"),
                Timestamp("2016-01-01 00:00:00", tz=tz, freq="30T"),
                Timestamp("2016-01-01 00:30:00", tz=tz, freq="30T"),
                Timestamp("2016-01-01 00:30:00", tz=tz, freq="30T"),
            ]
        )

        res = rng.repeat(reps)
        tm.assert_index_equal(res, expected_rng)
        assert res.freq is None

        tm.assert_index_equal(np.repeat(rng, reps), expected_rng)
        with pytest.raises(ValueError, match=msg):
            np.repeat(rng, reps, axis=1)

    def test_resolution(self, tz_naive_fixture):
        tz = tz_naive_fixture
        for freq, expected in zip(
            ["A", "Q", "M", "D", "H", "T", "S", "L", "U"],
            [
                "day",
                "day",
                "day",
                "day",
                "hour",
                "minute",
                "second",
                "millisecond",
                "microsecond",
            ],
        ):
            idx = pd.date_range(start="2013-04-01", periods=30, freq=freq, tz=tz)
            assert idx.resolution == expected

    def test_value_counts_unique(self, tz_naive_fixture):
        tz = tz_naive_fixture
        # GH 7735
        idx = pd.date_range("2011-01-01 09:00", freq="H", periods=10)
        # create repeated values, 'n'th element is repeated by n+1 times
        idx = DatetimeIndex(np.repeat(idx.values, range(1, len(idx) + 1)), tz=tz)

        exp_idx = pd.date_range("2011-01-01 18:00", freq="-1H", periods=10, tz=tz)
        expected = Series(range(10, 0, -1), index=exp_idx, dtype="int64")

        for obj in [idx, Series(idx)]:
            tm.assert_series_equal(obj.value_counts(), expected)

        expected = pd.date_range("2011-01-01 09:00", freq="H", periods=10, tz=tz)
        tm.assert_index_equal(idx.unique(), expected)

        idx = DatetimeIndex(
            [
                "2013-01-01 09:00",
                "2013-01-01 09:00",
                "2013-01-01 09:00",
                "2013-01-01 08:00",
                "2013-01-01 08:00",
                pd.NaT,
            ],
            tz=tz,
        )

        exp_idx = DatetimeIndex(["2013-01-01 09:00", "2013-01-01 08:00"], tz=tz)
        expected = Series([3, 2], index=exp_idx)

        for obj in [idx, Series(idx)]:
            tm.assert_series_equal(obj.value_counts(), expected)

        exp_idx = DatetimeIndex(["2013-01-01 09:00", "2013-01-01 08:00", pd.NaT], tz=tz)
        expected = Series([3, 2, 1], index=exp_idx)

        for obj in [idx, Series(idx)]:
            tm.assert_series_equal(obj.value_counts(dropna=False), expected)

        tm.assert_index_equal(idx.unique(), exp_idx)

    def test_nonunique_contains(self):
        # GH 9512
        for idx in map(
            DatetimeIndex,
            (
                [0, 1, 0],
                [0, 0, -1],
                [0, -1, -1],
                ["2015", "2015", "2016"],
                ["2015", "2015", "2014"],
            ),
        ):
            assert idx[0] in idx

    @pytest.mark.parametrize(
        "idx",
        [
            DatetimeIndex(
                ["2011-01-01", "2011-01-02", "2011-01-03"], freq="D", name="idx"
            ),
            DatetimeIndex(
                ["2011-01-01 09:00", "2011-01-01 10:00", "2011-01-01 11:00"],
                freq="H",
                name="tzidx",
                tz="Asia/Tokyo",
            ),
        ],
    )
    def test_order_with_freq(self, idx):
        ordered = idx.sort_values()
        tm.assert_index_equal(ordered, idx)
        assert ordered.freq == idx.freq

        ordered = idx.sort_values(ascending=False)
        expected = idx[::-1]
        tm.assert_index_equal(ordered, expected)
        assert ordered.freq == expected.freq
        assert ordered.freq.n == -1

        ordered, indexer = idx.sort_values(return_indexer=True)
        tm.assert_index_equal(ordered, idx)
        tm.assert_numpy_array_equal(indexer, np.array([0, 1, 2]), check_dtype=False)
        assert ordered.freq == idx.freq

        ordered, indexer = idx.sort_values(return_indexer=True, ascending=False)
        expected = idx[::-1]
        tm.assert_index_equal(ordered, expected)
        tm.assert_numpy_array_equal(indexer, np.array([2, 1, 0]), check_dtype=False)
        assert ordered.freq == expected.freq
        assert ordered.freq.n == -1

    @pytest.mark.parametrize(
        "index_dates,expected_dates",
        [
            (
                ["2011-01-01", "2011-01-03", "2011-01-05", "2011-01-02", "2011-01-01"],
                ["2011-01-01", "2011-01-01", "2011-01-02", "2011-01-03", "2011-01-05"],
            ),
            (
                ["2011-01-01", "2011-01-03", "2011-01-05", "2011-01-02", "2011-01-01"],
                ["2011-01-01", "2011-01-01", "2011-01-02", "2011-01-03", "2011-01-05"],
            ),
            (
                [pd.NaT, "2011-01-03", "2011-01-05", "2011-01-02", pd.NaT],
                [pd.NaT, pd.NaT, "2011-01-02", "2011-01-03", "2011-01-05"],
            ),
        ],
    )
    def test_order_without_freq(self, index_dates, expected_dates, tz_naive_fixture):
        tz = tz_naive_fixture

        # without freq
        index = DatetimeIndex(index_dates, tz=tz, name="idx")
        expected = DatetimeIndex(expected_dates, tz=tz, name="idx")

        ordered = index.sort_values()
        tm.assert_index_equal(ordered, expected)
        assert ordered.freq is None

        ordered = index.sort_values(ascending=False)
        tm.assert_index_equal(ordered, expected[::-1])
        assert ordered.freq is None

        ordered, indexer = index.sort_values(return_indexer=True)
        tm.assert_index_equal(ordered, expected)

        exp = np.array([0, 4, 3, 1, 2])
        tm.assert_numpy_array_equal(indexer, exp, check_dtype=False)
        assert ordered.freq is None

        ordered, indexer = index.sort_values(return_indexer=True, ascending=False)
        tm.assert_index_equal(ordered, expected[::-1])

        exp = np.array([2, 1, 3, 4, 0])
        tm.assert_numpy_array_equal(indexer, exp, check_dtype=False)
        assert ordered.freq is None

    def test_drop_duplicates_metadata(self):
        # GH 10115
        idx = pd.date_range("2011-01-01", "2011-01-31", freq="D", name="idx")
        result = idx.drop_duplicates()
        tm.assert_index_equal(idx, result)
        assert idx.freq == result.freq

        idx_dup = idx.append(idx)
        assert idx_dup.freq is None  # freq is reset
        result = idx_dup.drop_duplicates()
        tm.assert_index_equal(idx, result)
        assert result.freq is None

    def test_drop_duplicates(self):
        # to check Index/Series compat
        base = pd.date_range("2011-01-01", "2011-01-31", freq="D", name="idx")
        idx = base.append(base[:5])

        res = idx.drop_duplicates()
        tm.assert_index_equal(res, base)
        res = Series(idx).drop_duplicates()
        tm.assert_series_equal(res, Series(base))

        res = idx.drop_duplicates(keep="last")
        exp = base[5:].append(base[:5])
        tm.assert_index_equal(res, exp)
        res = Series(idx).drop_duplicates(keep="last")
        tm.assert_series_equal(res, Series(exp, index=np.arange(5, 36)))

        res = idx.drop_duplicates(keep=False)
        tm.assert_index_equal(res, base[5:])
        res = Series(idx).drop_duplicates(keep=False)
        tm.assert_series_equal(res, Series(base[5:], index=np.arange(5, 31)))

    @pytest.mark.parametrize(
        "freq",
        [
            "A",
            "2A",
            "-2A",
            "Q",
            "-1Q",
            "M",
            "-1M",
            "D",
            "3D",
            "-3D",
            "W",
            "-1W",
            "H",
            "2H",
            "-2H",
            "T",
            "2T",
            "S",
            "-3S",
        ],
    )
    def test_infer_freq(self, freq):
        # GH 11018
        idx = pd.date_range("2011-01-01 09:00:00", freq=freq, periods=10)
        result = pd.DatetimeIndex(idx.asi8, freq="infer")
        tm.assert_index_equal(idx, result)
        assert result.freq == freq

    def test_nat(self, tz_naive_fixture):
        tz = tz_naive_fixture
        assert pd.DatetimeIndex._na_value is pd.NaT
        assert pd.DatetimeIndex([])._na_value is pd.NaT

        idx = pd.DatetimeIndex(["2011-01-01", "2011-01-02"], tz=tz)
        assert idx._can_hold_na

        tm.assert_numpy_array_equal(idx._isnan, np.array([False, False]))
        assert idx.hasnans is False
        tm.assert_numpy_array_equal(idx._nan_idxs, np.array([], dtype=np.intp))

        idx = pd.DatetimeIndex(["2011-01-01", "NaT"], tz=tz)
        assert idx._can_hold_na

        tm.assert_numpy_array_equal(idx._isnan, np.array([False, True]))
        assert idx.hasnans is True
        tm.assert_numpy_array_equal(idx._nan_idxs, np.array([1], dtype=np.intp))

    def test_equals(self):
        # GH 13107
        idx = pd.DatetimeIndex(["2011-01-01", "2011-01-02", "NaT"])
        assert idx.equals(idx)
        assert idx.equals(idx.copy())
        assert idx.equals(idx.astype(object))
        assert idx.astype(object).equals(idx)
        assert idx.astype(object).equals(idx.astype(object))
        assert not idx.equals(list(idx))
        assert not idx.equals(pd.Series(idx))

        idx2 = pd.DatetimeIndex(["2011-01-01", "2011-01-02", "NaT"], tz="US/Pacific")
        assert not idx.equals(idx2)
        assert not idx.equals(idx2.copy())
        assert not idx.equals(idx2.astype(object))
        assert not idx.astype(object).equals(idx2)
        assert not idx.equals(list(idx2))
        assert not idx.equals(pd.Series(idx2))

        # same internal, different tz
        idx3 = pd.DatetimeIndex._simple_new(idx.asi8, tz="US/Pacific")
        tm.assert_numpy_array_equal(idx.asi8, idx3.asi8)
        assert not idx.equals(idx3)
        assert not idx.equals(idx3.copy())
        assert not idx.equals(idx3.astype(object))
        assert not idx.astype(object).equals(idx3)
        assert not idx.equals(list(idx3))
        assert not idx.equals(pd.Series(idx3))

    @pytest.mark.parametrize("values", [["20180101", "20180103", "20180105"], []])
    @pytest.mark.parametrize("freq", ["2D", Day(2), "2B", BDay(2), "48H", Hour(48)])
    @pytest.mark.parametrize("tz", [None, "US/Eastern"])
    def test_freq_setter(self, values, freq, tz):
        # GH 20678
        idx = DatetimeIndex(values, tz=tz)

        # can set to an offset, converting from string if necessary
        idx.freq = freq
        assert idx.freq == freq
        assert isinstance(idx.freq, ABCDateOffset)

        # can reset to None
        idx.freq = None
        assert idx.freq is None

    def test_freq_setter_errors(self):
        # GH 20678
        idx = DatetimeIndex(["20180101", "20180103", "20180105"])

        # setting with an incompatible freq
        msg = (
            "Inferred frequency 2D from passed values does not conform to "
            "passed frequency 5D"
        )
        with pytest.raises(ValueError, match=msg):
            idx.freq = "5D"

        # setting with non-freq string
        with pytest.raises(ValueError, match="Invalid frequency"):
            idx.freq = "foo"

    def test_offset_deprecated(self):
        # GH 20716
        idx = pd.DatetimeIndex(["20180101", "20180102"])

        # getter deprecated
        with tm.assert_produces_warning(FutureWarning):
            idx.offset

        # setter deprecated
        with tm.assert_produces_warning(FutureWarning):
            idx.offset = BDay()

class TestBusinessDatetimeIndex:
    def setup_method(self, method):
        self.rng = bdate_range(START, END)

    def test_comparison(self):
        d = self.rng[10]

        comp = self.rng > d
        assert comp[11]
        assert not comp[9]

    def test_pickle_unpickle(self):
        unpickled = tm.round_trip_pickle(self.rng)
        assert unpickled.freq is not None

    def test_copy(self):
        cp = self.rng.copy()
        repr(cp)
        tm.assert_index_equal(cp, self.rng)

    def test_shift(self):
        shifted = self.rng.shift(5)
        assert shifted[0] == self.rng[5]
        assert shifted.freq == self.rng.freq

        shifted = self.rng.shift(-5)
        assert shifted[5] == self.rng[0]
        assert shifted.freq == self.rng.freq

        shifted = self.rng.shift(0)
        assert shifted[0] == self.rng[0]
        assert shifted.freq == self.rng.freq

        rng = date_range(START, END, freq=BMonthEnd())
        shifted = rng.shift(1, freq=BDay())
        assert shifted[0] == rng[0] + BDay()

    def test_equals(self):
        assert not self.rng.equals(list(self.rng))

    def test_identical(self):
        t1 = self.rng.copy()
        t2 = self.rng.copy()
        assert t1.identical(t2)

        # name
        t1 = t1.rename("foo")
        assert t1.equals(t2)
        assert not t1.identical(t2)
        t2 = t2.rename("foo")
        assert t1.identical(t2)

        # freq
        t2v = Index(t2.values)
        assert t1.equals(t2v)
        assert not t1.identical(t2v)

class TestCustomDatetimeIndex:
    def setup_method(self, method):
        self.rng = bdate_range(START, END, freq="C")

    def test_comparison(self):
        d = self.rng[10]

        comp = self.rng > d
        assert comp[11]
        assert not comp[9]

    def test_copy(self):
        cp = self.rng.copy()
        repr(cp)
        tm.assert_index_equal(cp, self.rng)

    def test_shift(self):

        shifted = self.rng.shift(5)
        assert shifted[0] == self.rng[5]
        assert shifted.freq == self.rng.freq

        shifted = self.rng.shift(-5)
        assert shifted[5] == self.rng[0]
        assert shifted.freq == self.rng.freq

        shifted = self.rng.shift(0)
        assert shifted[0] == self.rng[0]
        assert shifted.freq == self.rng.freq

        with warnings.catch_warnings(record=True):
            warnings.simplefilter("ignore", pd.errors.PerformanceWarning)
            rng = date_range(START, END, freq=BMonthEnd())
            shifted = rng.shift(1, freq=CDay())
            assert shifted[0] == rng[0] + CDay()

    def test_shift_periods(self):
        # GH#22458 : argument 'n' was deprecated in favor of 'periods'
        idx = pd.date_range(start=START, end=END, periods=3)
        tm.assert_index_equal(idx.shift(periods=0), idx)
        tm.assert_index_equal(idx.shift(0), idx)
        with tm.assert_produces_warning(FutureWarning, check_stacklevel=True):
            tm.assert_index_equal(idx.shift(n=0), idx)

    def test_pickle_unpickle(self):
        unpickled = tm.round_trip_pickle(self.rng)
        assert unpickled.freq is not None

    def test_equals(self):
        assert not self.rng.equals(list(self.rng))


"""
Plot the drag coefficient over 4 oscillation cycles.
"""

import pathlib
import numpy
from matplotlib import pyplot
from scipy import signal

# Read the drag force from file.
simu_dir = pathlib.Path(__file__).parents[1]
data_dir = simu_dir / 'output'
filepath = data_dir / 'forces-0.txt'
with open(filepath, 'r') as infile:
    t, fx = numpy.loadtxt(infile, dtype=numpy.float64,
                          unpack=True, usecols=(0, 1))

# Set the parameters of the kinematics.
KC = 5.0  # Keulegan-Carpenter number
D = 1.0  # cylinder diameter
f = 0.2  # frequency of oscillation
w = 2 * numpy.pi * f  # angular frequency
Am = KC * D / (2 * numpy.pi)  # amplitude of oscillation
rho = 1.0  # fluid density
Um = w * Am  # maximum translational velocity of cylinder
V = numpy.pi * D**2 / 4  # volume of cylinder

# Add force due to body acceleration.
ax = w**2 * Am * numpy.sin(w * t)
fx += rho * V * ax
# Get the drag coefficient.
cd = fx / (0.5 * rho * Um**2 * D)

# Compute and print info abount extrema of the drag coefficient.
idx_min = signal.argrelextrema(fx, numpy.less_equal, order=100)[0][1:-1]
t_min = t[idx_min]
print('Non-dimensional time-interval between minima:\n\t{}'
      .format(f * (t_min[1:] - t_min[:-1])))
cd_min = cd[idx_min]
print('Drag coefficient valleys: {}'.format(cd_min))
idx_max = signal.argrelextrema(fx, numpy.greater_equal, order=100)[0][1:]
t_max = t[idx_max]
print('Non-dimensional time-interval between maxima:\n\t{}'
      .format(f * (t_max[1:] - t_max[:-1])))
cd_max = cd[idx_max]
print('Drag coefficient peaks: {}'.format(cd_max))

# Plot the drag coefficient over the 4 cycles.
pyplot.rcParams['font.size'] = 16
pyplot.rcParams['font.family'] = 'serif'
fig, ax = pyplot.subplots(figsize=(8.0, 4.0))
ax.grid()
ax.set_xlabel('$t / T$')
ax.set_ylabel('$C_D$')
ax.plot(f * t, cd)
ax.axis((0.0, 4.0, -6.0, 6.0))
fig.tight_layout()
pyplot.show()

# Save the figure.
fig_dir = simu_dir / 'figures'
fig_dir.mkdir(parents=True, exist_ok=True)
filepath = fig_dir / 'dragCoefficient.png'
fig.savefig(str(filepath), dpi=300)


"""
This is a setup.py script generated by py2applet

Usage:
    python3 setup-mac3.py py2app
"""

# manually remove sample-data mpl subdirectory from Python installation:
# sudo rm -rf /Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/matplotlib/mpl-data/sample_data

from distutils import sysconfig
their_parse_makefile = sysconfig.parse_makefile
def my_parse_makefile(filename, g):
    their_parse_makefile(filename, g)
    g['MACOSX_DEPLOYMENT_TARGET'] = '10.6'
sysconfig.parse_makefile = my_parse_makefile

import sys, os
from setuptools import setup

import string
from plistlib import Plist

import artisanlib

# current version of Artisan
VERSION = artisanlib.__version__
LICENSE = 'GNU General Public License (GPL)'

QTDIR = r'/Developer/Applications/Qt/'

APP = ['artisan.py']

DATA_FILES = [
    "LICENSE.txt",
    ("../Resources/qt_plugins/iconengines", [QTDIR + r'/plugins/iconengines/libqsvgicon.dylib']),
    ("../Resources/qt_plugins/imageformats", [QTDIR + r'/plugins/imageformats/libqsvg.dylib']),
    ("../Resources/qt_plugins/imageformats", [QTDIR + r'/plugins/imageformats/libqjpeg.dylib']),
    ("../Resources/qt_plugins/imageformats", [QTDIR + r'/plugins/imageformats/libqgif.dylib']),
    ("../Resources/qt_plugins/imageformats", [QTDIR + r'/plugins/imageformats/libqtiff.dylib']),
# standard QT translation needed to get the Application menu bar and
# the standard dialog elements translated
    ("../translations", [QTDIR + r'/translations/qt_de.qm']),
    ("../translations", [QTDIR + r'/translations/qt_es.qm']),
    ("../translations", [QTDIR + r'/translations/qt_fr.qm']),
    ("../translations", [QTDIR + r'/translations/qt_sv.qm']),
    ("../translations", [QTDIR + r'/translations/qt_zh_CN.qm']),
    ("../translations", [QTDIR + r'/translations/qt_zh_TW.qm']),
    ("../translations", [QTDIR + r'/translations/qt_ko.qm']),
    ("../translations", [QTDIR + r'/translations/qt_pt.qm']),
    ("../translations", [QTDIR + r'/translations/qt_ru.qm']),
    ("../translations", [QTDIR + r'/translations/qt_ar.qm']),
    ("../translations", [QTDIR + r'/translations/qt_ja.qm']),
    ("../translations", [QTDIR + r'/translations/qt_hu.qm']),
    ("../translations", [r"translations/artisan_de.qm"]),
    ("../translations", [r"translations/artisan_es.qm"]),
    ("../translations", [r"translations/artisan_fr.qm"]),
    ("../translations", [r"translations/artisan_sv.qm"]),
    ("../translations", [r'translations/artisan_zh_CN.qm']),
    ("../translations", [r'translations/artisan_zh_TW.qm']),
    ("../translations", [r'translations/artisan_ko.qm']),
    ("../translations", [r'translations/artisan_pt.qm']),
    ("../translations", [r'translations/artisan_ru.qm']),
    ("../translations", [r'translations/artisan_ar.qm']),
    ("../translations", [r"translations/artisan_it.qm"]),
    ("../translations", [r"translations/artisan_el.qm"]),
    ("../translations", [r"translations/artisan_no.qm"]),
    ("../translations", [r"translations/artisan_nl.qm"]),
    ("../translations", [r"translations/artisan_fi.qm"]),
    ("../translations", [r"translations/artisan_tr.qm"]),
    ("../translations", [r"translations/artisan_ja.qm"]),
    ("../translations", [r"translations/artisan_hu.qm"]),
    ("../translations", [r"translations/artisan_he.qm"]),
    ("../Resources", [r"qt.conf"]),
    ("../Resources", [r"artisanProfile.icns"]),
    ("../Resources", [r"artisanAlarms.icns"]),
    ("../Resources", [r"artisanPalettes.icns"]),
    ("../Resources", [r"artisanWheel.icns"]),
    ("../Resources", [r"includes/Humor-Sans.ttf"]),
  ]

plist = Plist.fromFile('Info3.plist')
plist.update({ 'CFBundleDisplayName': 'Artisan',
                    'CFBundleGetInfoString': 'Artisan, Roast Logger',
                    'CFBundleIdentifier': 'com.google.code.p.Artisan',
                    'CFBundleShortVersionString': VERSION,
                    'CFBundleVersion': 'Artisan ' + VERSION,
                    'LSMinimumSystemVersion': '10.6',
                    'LSMultipleInstancesProhibited': 'false',
                    'LSPrefersPPC': False,
                    'LSArchitecturePriority': 'x86_64',
                    'NSHumanReadableCopyright': LICENSE,
                })

OPTIONS = {
    'strip':True,
    'argv_emulation': False, # this would confuses GUI processing
    'semi_standalone': False,
    'site_packages': True,
    'dylib_excludes': ['phonon','QtDBus','QtDeclarative','QtDesigner',
                    'QtHelp','QtMultimedia','QtNetwork',
                    'QtOpenGL','QtScript','QtScriptTools',
                    'QtSql','QtTest','QtXmlPatterns','QtWebKit'],
#    'packages': ['matplotlib'], # with this the full pkg is copied to Resources/lib/python3.4
    'packages': ['yoctopuce'],
    'optimize':  2,
    'compressed': True,
    'iconfile': 'artisan.icns',
    'arch': 'x86_64',
    'matplotlib_backends': '-', # '-' for only-imported or explicit 'qt4agg'; without this the full pkg is copied to Resources/lib/python3.4
    'includes': ['serial',
                 'PyQt4',
                 'PyQt4.QtCore',
                 'PyQt4.QtGui',
                 'PyQt4.QtSvg',
                 'PyQt4.QtXml'],
    'excludes' :  ['_tkagg','_ps','_fltkagg','Tkinter','Tkconstants',
                      '_agg','_cairo','_gtk','gtkcairo','pydoc','sqlite3',
                      'bsddb','curses','tcl',
                      '_wxagg','_gtagg','_cocoaagg','_wx'],
    'plist'    : plist}

setup(
    name='Artisan',
    version=VERSION,
    author='YOUcouldbeTOO',
    author_email='zaub.ERASE.org@yahoo.com',
    license=LICENSE,
    app=APP,
    data_files=DATA_FILES,
    options={'py2app': OPTIONS},
    setup_requires=['py2app']
)

os.system(r'cp README.txt dist')
os.system(r'cp LICENSE.txt dist')
os.system(r'mkdir dist/Wheels')
os.system(r'mkdir dist/Wheels/Cupping')
os.system(r'mkdir dist/Wheels/Other')
os.system(r'mkdir dist/Wheels/Roasting')
os.system(r'cp Wheels/Cupping/* dist/Wheels/Cupping')
os.system(r'cp Wheels/Other/* dist/Wheels/Other')
os.system(r'cp Wheels/Roasting/* dist/Wheels/Roasting')
os.chdir('./dist')

# to prevent the error "Artisan.app/Contents/Resources/lib/python3.3/config-3.3m/Makefile'" on startup
# generated by v0.8 of py2app:
#os.system(r'cp /Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/config-3.3m/Makefile ./Artisan.app/Contents/Resources/lib/python3.3/config-3.3m/')

# delete unused Qt.framework files (py2app exclude does not seem to work)
print('*** Removing unused Qt frameworks ***')
for fw in [
            'phonon',
            'QtDeclarative',
            'QtHelp',
            'QtMultimedia',
            'QtNetwork',
            'QtOpenGL',
            'QtScript',
            'QtScriptTools',
            'QtSql',
            'QtTest',
            'QtWebKit',
            'QtXMLPatterns']:
    for root,dirs,files in os.walk('./Artisan.app/Contents/Frameworks/' + fw + ".framework"):
        for file in files:
            print('Deleting', file)
            os.remove(os.path.join(root,file))

print('*** Removing Qt debug libs ***')
for root, dirs, files in os.walk('.'):
    for file in files:
        if 'debug' in file:
            print('Deleting', file)
            os.remove(os.path.join(root,file))
        elif file.startswith('test_'):
            print('Deleting', file)
            os.remove(os.path.join(root,file))
        elif '_tests' in file:
            print('Deleting', file)
            os.remove(os.path.join(root,file))
        elif file.endswith('.pyc') and file != "site.pyc":
            print('Deleting', file)
            os.remove(os.path.join(root,file))
        # remove also all .h .in .cpp .cc .html files
        elif file.endswith('.h') and file != "pyconfig.h":
            print('Deleting', file)
            os.remove(os.path.join(root,file))
        elif file.endswith('.in'):
            print('Deleting', file)
            os.remove(os.path.join(root,file))
        elif file.endswith('.cpp'):
            print('Deleting', file)
            os.remove(os.path.join(root,file))
        elif file.endswith('.cc'):
            print('Deleting', file)
            os.remove(os.path.join(root,file))
# .afm files should not be removed as without matplotlib will fail on startup
#        elif file.endswith('.afm'):
#            print('Deleting', file)
#            os.remove(os.path.join(root,file))
    # remove test files
    for dir in dirs:
        if 'tests' in dir:
            for r,d,f in os.walk(os.path.join(root,dir)):
                for fl in f:
                    print('Deleting', os.path.join(r,fl))
                    os.remove(os.path.join(r,fl))

os.chdir('..')
os.system(r"rm artisan-mac-" + VERSION + r".dmg")
os.system(r'hdiutil create artisan-mac-' + VERSION + r'.dmg -volname "Artisan" -fs HFS+ -srcfolder "dist"')
# otool -L dist/Artisan.app/Contents/MacOS/Artisan


"""
Testing for the base module (sklearn.ensemble.base).
"""

# Authors: Gilles Louppe
# License: BSD 3 clause

from numpy.testing import assert_equal
from nose.tools import assert_true

from sklearn.utils.testing import assert_raise_message
from sklearn.datasets import load_iris
from sklearn.ensemble import BaggingClassifier
from sklearn.linear_model import Perceptron

def test_base():
    # Check BaseEnsemble methods.
    ensemble = BaggingClassifier(base_estimator=Perceptron(), n_estimators=3)

    iris = load_iris()
    ensemble.fit(iris.data, iris.target)
    ensemble.estimators_ = []  # empty the list and create estimators manually

    ensemble._make_estimator()
    ensemble._make_estimator()
    ensemble._make_estimator()
    ensemble._make_estimator(append=False)

    assert_equal(3, len(ensemble))
    assert_equal(3, len(ensemble.estimators_))

    assert_true(isinstance(ensemble[0], Perceptron))

def test_base_zero_n_estimators():
    # Check that instantiating a BaseEnsemble with n_estimators<=0 raises
    # a ValueError.
    ensemble = BaggingClassifier(base_estimator=Perceptron(), n_estimators=0)
    iris = load_iris()
    assert_raise_message(ValueError,
                         "n_estimators must be greater than zero, got 0.",
                         ensemble.fit, iris.data, iris.target)


"""
=========================================================
Compute power spectrum densities of the sources with dSPM
=========================================================

Returns an STC file containing the PSD (in dB) of each of the sources.

"""
# Authors: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#
# License: BSD (3-clause)

import matplotlib.pyplot as plt

import mne
from mne import io
from mne.datasets import sample
from mne.minimum_norm import read_inverse_operator, compute_source_psd

print(__doc__)

###############################################################################
# Set parameters
data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_raw.fif'
fname_inv = data_path + '/MEG/sample/sample_audvis-meg-oct-6-meg-inv.fif'
fname_label = data_path + '/MEG/sample/labels/Aud-lh.label'

# Setup for reading the raw data
raw = io.Raw(raw_fname, verbose=False)
events = mne.find_events(raw, stim_channel='STI 014')
inverse_operator = read_inverse_operator(fname_inv)
raw.info['bads'] = ['MEG 2443', 'EEG 053']

# picks MEG gradiometers
picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=True,
                       stim=False, exclude='bads')

tmin, tmax = 0, 120  # use the first 120s of data
fmin, fmax = 4, 100  # look at frequencies between 4 and 100Hz
n_fft = 2048  # the FFT size (n_fft). Ideally a power of 2
label = mne.read_label(fname_label)

stc = compute_source_psd(raw, inverse_operator, lambda2=1. / 9., method="dSPM",
                         tmin=tmin, tmax=tmax, fmin=fmin, fmax=fmax,
                         pick_ori="normal", n_fft=n_fft, label=label)

stc.save('psd_dSPM')

###############################################################################
# View PSD of sources in label
plt.plot(1e3 * stc.times, stc.data.T)
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD (dB)')
plt.title('Source Power Spectrum (PSD)')
plt.show()


import warnings

__all__ = ["deprecated", ]

class deprecated(object):
    """Decorator to mark a function or class as deprecated.

    Issue a warning when the function is called/the class is instantiated and
    adds a warning to the docstring.

    The optional extra argument will be appended to the deprecation message
    and the docstring. Note: to use this with the default value for extra, put
    in an empty of parentheses:

    >>> from sklearn.utils import deprecated
    >>> deprecated() # doctest: +ELLIPSIS
    <sklearn.utils.deprecation.deprecated object at ...>

    >>> @deprecated()
    ... def some_function(): pass
    """

    # Adapted from http://wiki.python.org/moin/PythonDecoratorLibrary,
    # but with many changes.

    def __init__(self, extra=''):
        """
        Parameters
        ----------
        extra : string
          to be added to the deprecation messages

        """
        self.extra = extra

    def __call__(self, obj):
        if isinstance(obj, type):
            return self._decorate_class(obj)
        else:
            return self._decorate_fun(obj)

    def _decorate_class(self, cls):
        msg = "Class %s is deprecated" % cls.__name__
        if self.extra:
            msg += "; %s" % self.extra

        # FIXME: we should probably reset __new__ for full generality
        init = cls.__init__

        def wrapped(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning)
            return init(*args, **kwargs)
        cls.__init__ = wrapped

        wrapped.__name__ = '__init__'
        wrapped.__doc__ = self._update_doc(init.__doc__)
        wrapped.deprecated_original = init

        return cls

    def _decorate_fun(self, fun):
        """Decorate function fun"""

        msg = "Function %s is deprecated" % fun.__name__
        if self.extra:
            msg += "; %s" % self.extra

        def wrapped(*args, **kwargs):
            warnings.warn(msg, category=DeprecationWarning)
            return fun(*args, **kwargs)

        wrapped.__name__ = fun.__name__
        wrapped.__dict__ = fun.__dict__
        wrapped.__doc__ = self._update_doc(fun.__doc__)

        return wrapped

    def _update_doc(self, olddoc):
        newdoc = "DEPRECATED"
        if self.extra:
            newdoc = "%s: %s" % (newdoc, self.extra)
        if olddoc:
            newdoc = "%s\n\n%s" % (newdoc, olddoc)
        return newdoc