# -*- coding: utf-8 -*-
"""
Created on Fri Oct 31 11:07:20 2014

@author: junz
"""
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
import matplotlib.colors as col
import scipy.ndimage as ni
import ImageAnalysis as ia

try:
    import skimage.external.tifffile as tf
except ImportError:
    import tifffile as tf

try: import cv2
except ImportError as e: print e

def get_rgb(colorStr):
    """
    get R,G,B int value from a hex color string
    """
    return int(colorStr[1:3], 16), int(colorStr[3:5], 16), int(colorStr[5:7], 16)

def get_color_str(R, G, B):
    """
    get hex color string from R,G,B value (integer with uint8 format)
    """
    if not (isinstance(R, (int, long)) and isinstance(G, (int, long)) and isinstance(G, (int, long))):
        raise TypeError, 'Input R, G and B should be integer!'

    if not ((0 <= R <= 255) and (0 <= G <= 255) and (
            0 <= B <= 255)): raise ValueError, 'Input R, G and B should between 0 and 255!'
    return '#' + ''.join(map(chr, (R, G, B))).encode('hex')

def binary_2_rgba(img, foregroundColor='#ff0000', backgroundColor='#000000', foregroundAlpha=255, backgroundAlpha=0):
    """
    generate display image in (RGBA).(np.uint8) format which can be displayed by imshow
    :param img: input image, should be a binary array (np.bool, or np.(u)int
    :param foregroundColor: color for 1 in the array, RGB str, i.e. '#ff0000'
    :param backgroundColor: color for 0 in the array, RGB str, i.e. '#ff00ff'
    :param foregroundAlpha: alpha for 1 in the array, int, 0-255
    :param backgroundAlpha: alpha for 1 in the array, int, 0-255
    :return: displayImg, (RGBA).(np.uint8) format, ready for imshow
    """

    if img.dtype == np.bool:
        pass
    elif issubclass(img.dtype.type, np.integer):
        if np.amin(img) < 0 or np.amax(img) > 1: raise ValueError, 'Values of input image should be either 0 or 1.'
    else:
        raise TypeError, 'Data type of input image should be either np.bool or integer.'

    if type(foregroundAlpha) is int:
        if foregroundAlpha < 0 or foregroundAlpha > 255: raise ValueError, 'Value of foreGroundAlpha should be between 0 and 255.'
    else:
        raise TypeError, 'Data type of foreGroundAlpha should be integer.'

    if type(backgroundAlpha) is int:
        if backgroundAlpha < 0 or backgroundAlpha > 255: raise ValueError, 'Value of backGroundAlpha should be between 0 and 255.'
    else:
        raise TypeError, 'Data type of backGroundAlpha should be integer.'

    fR, fG, fB = get_rgb(foregroundColor)
    bR, bG, bB = get_rgb(backgroundColor)

    displayImg = np.zeros((img.shape[0], img.shape[1], 4)).astype(np.uint8)
    displayImg[img == 1] = np.array([fR, fG, fB, foregroundAlpha]).astype(np.uint8)
    displayImg[img == 0] = np.array([bR, bG, bB, backgroundAlpha]).astype(np.uint8)

    return displayImg

def scalar_2_rgba(img, color='#ff0000'):
    """
    generate display a image in (RGBA).(np.uint8) format which can be displayed by imshow
    alpha is defined by values in the img
    :param img: input image
    :param alphaMatrix: matrix of alpha
    :param foreGroundColor: color for 1 in the array, RGB str, i.e. '#ff0000'
    :return: displayImg, (RGBA).(np.uint8) format, ready for imshow
    """

    R, G, B = get_rgb(color)

    RMatrix = (R * ia.array_nor(img.astype(np.float32))).astype(np.uint8)
    GMatrix = (G * ia.array_nor(img.astype(np.float32))).astype(np.uint8)
    BMatrix = (B * ia.array_nor(img.astype(np.float32))).astype(np.uint8)

    alphaMatrix = (ia.array_nor(img.astype(np.float32)) * 255).astype(np.uint8)

    displayImg = np.zeros((img.shape[0], img.shape[1], 4)).astype(np.uint8)
    displayImg[:, :, 0] = RMatrix;
    displayImg[:, :, 1] = GMatrix;
    displayImg[:, :, 2] = BMatrix;
    displayImg[:, :, 3] = alphaMatrix

    return displayImg

def bar_graph(left,
              height,
              error,
              errorDir='both',  # 'both', 'positive' or 'negative'
              width=0.1,
              plotAxis=None,
              lw=3,
              faceColor='#000000',
              edgeColor='none',
              capSize=10,
              label=None
              ):
    """
    plot a single bar with error bar
    """

    if not plotAxis:
        f = plt.figure()
        plotAxis = f.add_subplot(111)

    if errorDir == 'both':
        yerr = error
    elif errorDir == 'positive':
        yerr = [[0], [error]]
    elif errorDir == 'negative':
        yerr = [[error], [0]]

    plotAxis.errorbar(left + width / 2,
                      height,
                      yerr=yerr,
                      lw=lw,
                      capsize=capSize,
                      capthick=lw,
                      color=edgeColor)

    plotAxis.bar(left,
                 height,
                 width=width,
                 color=faceColor,
                 edgecolor=edgeColor,
                 lw=lw,
                 label=label)

    return plotAxis

def random_color(numOfColor=10):
    """
    generate as list of random colors
    """
    numOfColor = int(numOfColor)

    colors = []

    Cmatrix = (np.random.rand(numOfColor, 3) * 255).astype(np.uint8)

    for i in range(numOfColor):

        r = hex(Cmatrix[i][0]).split('x')[1]
        if len(r) == 1:
            r = '0' + r

        g = hex(Cmatrix[i][1]).split('x')[1]
        if len(g) == 1:
            g = '0' + g

        b = hex(Cmatrix[i][2]).split('x')[1]
        if len(b) == 1:
            b = '0' + b

        colors.append('#' + r + g + b)

    return colors

def show_movie(path,  # tif file path or numpy arrary of the movie
               mode='raw',  # 'raw', 'dF' or 'dFoverF'
               baselinePic=None,  # picuture of baseline
               baselineType='mean',  # way to calculate baseline
               cmap='gray'):
    """
    plot tf movie in the way defined by mode
    """

    if isinstance(path, str):
        rawMov = tf.imread(path)
    elif isinstance(path, np.ndarray):
        rawMov = path

    if mode == 'raw':
        mov = rawMov
    else:
        _, dFMov, dFoverFMov = ia.normalize_movie(rawMov,
                                                  baselinePic=baselinePic,
                                                  baselineType=baselineType)
        if mode == 'dF':
            mov = dFMov
        elif mode == 'dFoverF':
            mov = dFoverFMov
        else:
            raise LookupError, 'The "mode" should be "raw", "dF" or "dFoverF"!'

    if isinstance(path, str):
        tf.imshow(mov,
                  cmap=cmap,
                  vmax=np.amax(mov),
                  vmin=np.amin(mov),
                  title=mode + ' movie of ' + path)
    elif isinstance(path, np.ndarray):
        tf.imshow(mov,
                  cmap=cmap,
                  vmax=np.amax(mov),
                  vmin=np.amin(mov),
                  title=mode + ' Movie')

    return mov

def standalone_color_bar(vmin, vmax, cmap, sectionNum=10):
    """
    plot a stand alone color bar.
    """

    a = np.array([[vmin, vmax]])

    plt.figure(figsize=(0.1, 9))

    img = plt.imshow(a, cmap=cmap, vmin=vmin, vmax=vmax)
    plt.gca().set_visible(False)
    cbar = plt.colorbar()
    cbar.set_ticks(np.linspace(vmin, vmax, num=sectionNum + 1))

def alpha_blending(image, alphaData, vmin, vmax, cmap='Paired', sectionNum=10, background=-1, interpolation='nearest',
                   isSave=False, savePath=None):
    """
    Generate image with transparency weighted by another matrix.

    Plot numpy array 'image' with colormap 'cmap'. And define the tranparency
    of each pixel by the value in another numpy array alphaData.

    All the elements in alphaData should be non-negative.
    """

    if image.shape != alphaData.shape:
        raise LookupError, '"image" and "alphaData" should have same shape!!'

    if np.amin(alphaData) < 0:
        raise ValueError, 'All the elements in alphaData should be bigger than zero.'

    # normalize image
    image[image > vmax] = vmax
    image[image < vmin] = vmin

    image = (image - vmin) / (vmax - vmin)

    # get colored image of image
    exec ('colorImage = cm.' + cmap + '(image)')

    # normalize alphadata
    alphaDataNor = alphaData / np.amax(alphaData)
    alphaDataNor = np.sqrt(alphaDataNor)

    colorImage[:, :, 3] = alphaDataNor

    # plt.figure()
    # plot dummy figure for colorbar
    a = np.array([[vmin, vmax]])
    plt.imshow(a, cmap=cmap, vmin=vmin, vmax=vmax, alpha=0)
    # plt.gca().set_visible(False)
    cbar = plt.colorbar()
    cbar.set_ticks(np.linspace(vmin, vmax, num=sectionNum + 1))
    cbar.set_alpha(1)
    cbar.draw_all()

    # generate black background
    b = np.array(colorImage)
    b[:] = background
    b[:, :, 3] = 1
    plt.imshow(b, cmap='gray')

    # plot map
    plt.imshow(colorImage, interpolation=interpolation)

    return colorImage

def plot_mask(mask, plotAxis=None, color='#ff0000', zoom=1, borderWidth=None, closingIteration=None):
    """
    plot mask borders in a given color
    """

    if not plotAxis:
        f = plt.figure()
        plotAxis = f.add_subplot(111)

    cmap1 = col.ListedColormap(color, 'temp')
    cm.register_cmap(cmap=cmap1)

    if zoom != 1:
        mask = ni.interpolation.zoom(mask, zoom, order=0)

    mask2 = mask.astype(np.float32)
    mask2[np.invert(np.isnan(mask2))] = 1.
    mask2[np.isnan(mask2)] = 0.

    struc = ni.generate_binary_structure(2, 2)
    if borderWidth:
        border = mask2 - ni.binary_erosion(mask2, struc, iterations=borderWidth).astype(np.float32)
    else:
        border = mask2 - ni.binary_erosion(mask2, struc).astype(np.float32)

    if closingIteration:
        border = ni.binary_closing(border, iterations=closingIteration).astype(np.float32)

    border[border == 0] = np.nan

    currfig = plotAxis.imshow(border, cmap='temp', interpolation='nearest')

    return currfig

def plot_mask_borders(mask, plotAxis=None, color='#ff0000', zoom=1, borderWidth=2, closingIteration=None, **kwargs):
    """
    plot mask (ROI) borders by using pyplot.contour function. all the 0s and Nans in the input mask will be considered
    as background, and non-zero, non-nan pixel will be considered in ROI.
    """
    if not plotAxis:
        f = plt.figure()
        plotAxis = f.add_subplot(111)

    plotingMask = np.ones(mask.shape, dtype=np.uint8)

    plotingMask[np.logical_or(np.isnan(mask), mask == 0)] = 0

    if zoom != 1:
        plotingMask = cv2.resize(plotingMask.astype(np.float),
                                 dsize=(int(plotingMask.shape[1] * zoom), int(plotingMask.shape[0] * zoom)))
        plotingMask[plotingMask < 0.5] = 0
        plotingMask[plotingMask >= 0.5] = 1
        plotingMask = plotingMask.astype(np.uint8)

    if closingIteration is not None:
        plotingMask = ni.binary_closing(plotingMask, iterations=closingIteration).astype(np.uint8)

    plotingMask = ni.binary_erosion(plotingMask, iterations=borderWidth)

    currfig = plotAxis.contour(plotingMask, levels=[0.5], colors=color, linewidths=borderWidth, **kwargs)

    # put y axis in decreasing order
    y_lim = list(plotAxis.get_ylim())
    y_lim.sort()
    plotAxis.set_ylim(y_lim[::-1])

    plotAxis.set_aspect('equal')

    return currfig

def grid_axis(rowNum, columnNum, totalPlotNum, **kwarg):
    """
    return figure handles and axis handels for multiple subplots and figures
    """

    figureNum = totalPlotNum // (rowNum * columnNum) + 1

    figureHandles = []

    for i in range(figureNum):
        f = plt.figure(**kwarg)
        figureHandles.append(f)

    axisHandles = []
    for i in range(totalPlotNum):
        currFig = figureHandles[i // (rowNum * columnNum)]
        currIndex = i % (rowNum * columnNum)
        currAxis = currFig.add_subplot(rowNum, columnNum, currIndex + 1)
        axisHandles.append(currAxis)

    return figureHandles, axisHandles

def tile_axis(f, rowNum, columnNum, topDownMargin=0.05, leftRightMargin=0.05, rowSpacing=0.05, columnSpacing=0.05):
    if 2 * topDownMargin + (
        (rowNum - 1) * rowSpacing) >= 1: raise ValueError, 'Top down margin or row spacing are too big!'
    if 2 * leftRightMargin + (
        (columnNum - 1) * columnSpacing) >= 1: raise ValueError, 'Left right margin or column spacing are too big!'

    height = (1 - (2 * topDownMargin) - (rowNum - 1) * rowSpacing) / rowNum
    width = (1 - (2 * leftRightMargin) - (columnNum - 1) * columnSpacing) / columnNum

    xStarts = np.arange(leftRightMargin, 1 - leftRightMargin, (width + columnSpacing))
    yStarts = np.arange(topDownMargin, 1 - topDownMargin, (height + rowSpacing))[::-1]

    axisList = [[f.add_axes([xStart, yStart, width, height]) for xStart in xStarts] for yStart in yStarts]

    return axisList

def save_figure_without_borders(f,
                                savePath,
                                removeSuperTitle=True,
                                **kwargs):
    """
    remove borders of a figure
    """
    f.gca().get_xaxis().set_visible(False)
    f.gca().get_yaxis().set_visible(False)
    f.gca().set_title('')
    if removeSuperTitle:
        f.suptitle('')
    f.savefig(savePath, pad_inches=0, bbox_inches='tight', **kwargs)

def merge_normalized_images(imgList, isFilter=True, sigma=50, mergeMethod='mean', dtype=np.float32):
    """
    merge images in a list in to one, for each image, local intensity variability will be removed by subtraction of
    gaussian filtered image. Then all images will be collapsed by the mergeMethod in to single image
    """

    imgList2 = []

    for currImg in imgList:
        imgList2.append(ia.array_nor(currImg.astype(dtype)))

    if mergeMethod == 'mean':
        mergedImg = np.mean(np.array(imgList2), axis=0)
    elif mergeMethod == 'min':
        mergedImg = np.min(np.array(imgList2), axis=0)
    elif mergeMethod == 'max':
        mergedImg = np.max(np.array(imgList2), axis=0)
    elif mergeMethod == 'median':
        mergedImg = np.median(np.array(imgList2), axis=0)

    if isFilter:
        mergedImgf = ni.filters.gaussian_filter(mergedImg.astype(np.float), sigma=sigma)
        return ia.array_nor(mergedImg - mergedImgf).astype(dtype)
    else:
        return ia.array_nor(mergedImg).astype(dtype)

# def hue2RGB(hue):
#     """
#     get the RGB value as format as hex string from the decimal ratio of hue (from 0 to 1)
#     color model as described in:
#     https://en.wikipedia.org/wiki/Hue
#     """
#     if hue < 0: hue = 0
#     if hue > 1: hue = 1
#     color = colorsys.hsv_to_rgb(hue,1,1)
#     color = [int(x*255) for x in color]
#     return get_color_str(*color)
#
#
def hot_2_rgb(hot):
    """
    get the RGB value as format as hex string from the decimal ratio of hot colormap (from 0 to 1)
    """
    if hot < 0: hot = 0
    if hot > 1: hot = 1
    cmap_hot = plt.get_cmap('hot')
    color = cmap_hot(hot)[0:3];
    color = [int(x * 255) for x in color]
    return get_color_str(*color)

def value_2_rgb(value, cmap):
    """
    get the RGB value as format as hex string from the decimal ratio of a given colormap (from 0 to 1)
    """
    if value < 0: value = 0
    if value > 1: value = 1
    cmap = plt.get_cmap(cmap)
    color = cmap(value)[0:3];
    color = [int(x * 255) for x in color]
    return get_color_str(*color)

if __name__ == '__main__':
    plt.ioff()
    print 'for debug'


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
import numpy as np
import pandas as pd

__author__ = 'Yuji Ikeda'
__version__ = '0.1.0'

def create_statistical_functions():
    return [
        ('sum', np.sum),
        ('avg.', np.average),
        ('s.d.', lambda x: np.std(x, ddof=0)),
        ('abs._sum', lambda x: np.sum(np.abs(x))),
        ('abs._avg.', lambda x: np.average(np.abs(x))),
        ('abs._s.d.', lambda x: np.std(np.abs(x), ddof=0)),
    ]

def create_data_stat(data, keys, properties):
    """

    :param data: pandas.DataFrame
    :param keys: List of strings
    :param properties: List of strings
    :return:
    """
    functions = create_statistical_functions()
    return data.groupby(keys, sort=False).agg(functions)[properties]

class Displacements(object):
    def __init__(self, atoms, atoms_ideal):
        self._atoms = atoms
        self._atoms_ideal = atoms_ideal
        self._initialize_data()

    def _initialize_data(self):
        data = pd.DataFrame()
        data['symbol'] = self._atoms.get_chemical_symbols()
        data['atom'] = ''
        data = data.reset_index()  # data['index'] is created
        self._data = data

    def run(self):
        self.calculate_displacements()
        self.write()

    def _calculate_origin(self):
        positions = self._atoms.get_scaled_positions()
        positions_ideal = self._atoms_ideal.get_scaled_positions()
        diff = positions - positions_ideal
        diff -= np.rint(diff)
        return np.average(diff, axis=0)

    def calculate_displacements(self):
        atoms = self._atoms
        atoms_ideal =self._atoms_ideal

        origin = self._calculate_origin()

        positions = atoms.get_scaled_positions()
        positions_ideal = atoms_ideal.get_scaled_positions()
        diff = positions - (positions_ideal + origin)
        diff -= np.rint(diff)
        diff = np.dot(diff, atoms.get_cell())  # frac -> A
        displacements = np.sqrt(np.sum(diff ** 2, axis=1))

        self._data['displacement'] = displacements

    def write(self):
        filename = self._create_filename()
        data = self._data

        properties = ['displacement']

        with open(filename, 'w') as f:
            f.write(self._create_header())
            f.write('{:<22s}{:<18s}'.format('#', 'displacements_(A)'))
            f.write('\n')
            for i, x in data.iterrows():
                f.write('atom ')
                f.write('{:11d}'.format(x['index']))
                f.write(' {:5s}'.format(x['symbol']))
                f.write('{:18.12f}'.format(x['displacement']))
                f.write('\n')

            f.write('\n')

            # Write statistics for all atoms
            data_stat = create_data_stat(data, 'atom', properties)
            for k0, x in data_stat.iterrows():
                for k1, v in x.iteritems():
                    f.write('{:16}'.format(k1[1]))
                    f.write(' {:5s}'.format(k0))
                    f.write('{:18.12f}'.format(v))
                    f.write('\n')
                f.write('\n')

            # Write statistics for each symbol
            data_stat = create_data_stat(data, 'symbol', properties)
            for k0, x in data_stat.iterrows():
                for k1, v in x.iteritems():
                    f.write('{:16s}'.format(k1[1]))
                    f.write(' {:5s}'.format(k0))
                    f.write('{:18.12f}'.format(v))
                    f.write('\n')
                f.write('\n')

    def _create_header(self):
        return ''

    def _create_filename(self):
        return 'displacements.dat'

    def get_data(self):
        return self._data


"""
==========================================
One-class SVM with non-linear kernel (RBF)
==========================================

An example using a one-class SVM for novelty detection.

:ref:`One-class SVM <svm_outlier_detection>` is an unsupervised
algorithm that learns a decision function for novelty detection:
classifying new data as similar or different to the training set.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.font_manager
from sklearn import svm

xx, yy = np.meshgrid(np.linspace(-5, 5, 500), np.linspace(-5, 5, 500))
# Generate train data
X = 0.3 * np.random.randn(100, 2)
X_train = np.r_[X + 2, X - 2]
# Generate some regular novel observations
X = 0.3 * np.random.randn(20, 2)
X_test = np.r_[X + 2, X - 2]
# Generate some abnormal novel observations
X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))

# fit the model
clf = svm.OneClassSVM(nu=0.1, kernel="rbf", gamma=0.1)
clf.fit(X_train)
y_pred_train = clf.predict(X_train)
y_pred_test = clf.predict(X_test)
y_pred_outliers = clf.predict(X_outliers)
n_error_train = y_pred_train[y_pred_train == -1].size
n_error_test = y_pred_test[y_pred_test == -1].size
n_error_outliers = y_pred_outliers[y_pred_outliers == 1].size

# plot the line, the points, and the nearest vectors to the plane
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)

plt.title("Novelty Detection")
plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), 0, 7), cmap=plt.cm.PuBu)
a = plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors='darkred')
plt.contourf(xx, yy, Z, levels=[0, Z.max()], colors='palevioletred')

s = 40
b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white', s=s)
b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='blueviolet', s=s)
c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='gold', s=s)
plt.axis('tight')
plt.xlim((-5, 5))
plt.ylim((-5, 5))
plt.legend([a.collections[0], b1, b2, c],
           ["learned frontier", "training observations",
            "new regular observations", "new abnormal observations"],
           loc="upper left",
           prop=matplotlib.font_manager.FontProperties(size=11))
plt.xlabel(
    "error train: %d/200 ; errors novel regular: %d/40 ; "
    "errors novel abnormal: %d/40"
    % (n_error_train, n_error_test, n_error_outliers))
plt.show()


"""
This demo demonstrates how to use a mean function in a GP and allow the model
to discover the most important basis functions.
This model is equivalent to a Relevance Vector Machine.

Author:
    Ilias Bilionis

Date:
    3/20/2014

"""

import numpy as np
import GPy
import pymcmc as pm
import matplotlib.pyplot as plt

# Write a class that represents the mean you wish to use:
class PolynomialBasis(object):
    """
    A simple set of polynomials.

    :param degree:  The degree of the polynomials.
    :type degree:   int
    """

    def __init__(self, degree):
        """
        The constructor can do anything you want. The object should be
        constructed before doing anything with pymcmc in any case.
        Just make sure that inside the constructor you define the ``num_output``
        attribute whose value should be equal to the number of basis functions.
        """
        self.degree = degree
        self.num_output = degree + 1 # YOU HAVE TO DEFINE THIS ATTRIBUTE!

    def __call__(self, X):
        """
        Evaluate the basis functions at ``X``.

        Now, you should assume that ``X`` is a 2D numpy array of size
        ``num_points x input_dim``. If ``input_dim`` is 1, then you still need
        to consider it as a 2D array because this is the kind of data that GPy
        requires. If you want to make the function work also with 1D arrays if
        ``input_dim`` is one the use the trick below.

        The output of this function should be the design matrix. That is,
        it should be the matrix ``phi`` of dimensions
        ``num_points x num_output``. In otherwors, ``phi[i, j]`` should be
        the value of basis function ``phi_j`` at ``X[i, :]``.
        """
        if X.ndim == 1:
            X = X[:, None] # Trick for 1D arrays
        return np.hstack([X ** i for i in range(self.degree + 1)])

# Pick your degree
degree = 5
# Construct your basis
poly_basis = PolynomialBasis(degree)
# Let us generate some random data to play with
# The number of input dimensions
input_dim = 1
# The number of observations
num_points = 50
# The noise level we are going to add to the observations
noise = 0.1
# Observed inputs
X = 20. * np.random.rand(num_points, 1) - 10.
# The observations we make
Y = np.sin(X) / X + noise * np.random.randn(num_points, 1) - 0.1 * X + 0.1 * X ** 3
# Let's construct a GP model with just a mean and a diagonal covariance
# This is the mean (and at the same time the kernel)
mean = pm.MeanFunction(input_dim, poly_basis, ARD=True)
# Add an RBF kernel
kernel = GPy.kern.RBF(input_dim)
# Now, let's construct the model
model = GPy.models.GPRegression(X, Y, kernel=mean + kernel)
print 'Model before training:'
print str(model)
# You may just train the model by maximizing the likelihood:
model.optimize_restarts(messages=True)
print 'Trained model:'
print str(model)
print model.add.mean.variance
# And just plot the predictions
model.plot(plot_limits=(-10, 15))
# Let us also plot the full function
x = np.linspace(-10, 15, 100)[:, None]
y = np.sin(x) / x - 0.1 * x + 0.1 * x ** 3
plt.plot(x, y, 'r', linewidth=2)
plt.legend(['Mean of GP', '5% percentile of GP', '95% percentile of GP',
            'Observations', 'Real Underlying Function'], loc='best')
plt.title('Model trained by maximizing the likelihood')
plt.show()
a = raw_input('press enter to continue...')
# Or you might want to do it using MCMC:
new_mean = pm.MeanFunction(input_dim, poly_basis, ARD=True)
new_kernel = GPy.kern.RBF(input_dim)
new_model = GPy.models.GPRegression(X, Y, kernel=mean + new_kernel)
proposal = pm.MALAProposal(dt=0.1)
mcmc = pm.MetropolisHastings(new_model, proposal=proposal)
mcmc.sample(50000, num_thin=100, num_burn=1000, verbose=True)
print 'Model trained with MCMC:'
print str(new_model)
print new_model.add.mean.variance
# Plot everything for this too:
new_model.plot(plot_limits=(-10., 15.))
# Let us also plot the full function
plt.plot(x, y, 'r', linewidth=2)
plt.legend(['Mean of GP', '5% percentile of GP', '95% percentile of GP',
            'Observations', 'Real Underlying Function'], loc='best')
plt.title('Model trained by MCMC')
plt.show()
a = raw_input('press enter to continue...')


import numpy as np
from sklearn.utils.validation import check_array

class NotFittedError(ValueError, AttributeError):
    pass

def check_feature_array(array, n_features=None):
    array = check_array(array, ensure_2d=True, allow_nd=False)
    if n_features is not None and array.shape[1] != n_features:
        raise ValueError('feature array must have exactly %d features' % n_features)
    return array

def check_is_fitted(estimator, attributes, msg=None, all_or_any=all):
    # Based on sklearn.util.validation.check_is_fitted but also ensures
    # that the attribute is not None
    if msg is None:
        msg = ("This %(name)s instance is not fitted yet. Call 'fit' with "
               "appropriate arguments before using this method.")

    if not hasattr(estimator, 'fit'):
        raise TypeError("%s is not an estimator instance." % estimator)

    if not isinstance(attributes, (list, tuple)):
        attributes = [attributes]

    if not all_or_any([hasattr(estimator, attr) for attr in attributes]):
        raise NotFittedError(msg % {'name': type(estimator).__name__})

    if not all_or_any([getattr(estimator, attr) for attr in attributes]):
        raise NotFittedError(msg % {'name': type(estimator).__name__})

def check_multilabel_array(array, n_labels=None, force_binary=True):
    array = check_array(array, ensure_2d=True, allow_nd=False, dtype=int)
    if n_labels is not None and array.shape[1] != n_labels:
        raise ValueError('multilabel array must have exactly %d labels' % n_labels)
    if force_binary:
        count_ones = np.count_nonzero(array == 1)
        count_zeros = np.count_nonzero(array == 0)
        if np.size(array) != count_ones + count_zeros:
            raise ValueError('multilabel array must be binary')
    return array

def pad_sequences(X):
    # Find longest sequence
    n_samples_max = 0
    for X_curr in X:
        n_samples_curr = X_curr.shape[0]
        if n_samples_curr > n_samples_max:
            n_samples_max = n_samples_curr

    # Adjust length of all sequences to be equal
    for idx, X_curr in enumerate(X):
        n_samples_curr = X_curr.shape[0]
        delta_samples = n_samples_max - n_samples_curr
        assert delta_samples >= 0
        if delta_samples > 0:
            fill_array = np.zeros((delta_samples, X_curr.shape[1]))
            X[idx] = np.append(X_curr, fill_array, axis=0)
            assert X[idx].shape[0] == n_samples_max
    X = np.asarray(X)
    return X


#!/usr/bin/python

import numpy as np
import matplotlib.pyplot as pyplot
from datetime import datetime
import os
import glob
import sys
import re
import argparse
import cv2
import random
import ast

# Argument parsing
#parser = argparse.ArgumentParser(description='Jumpball analyze')
#parser.add_argument('-s', '--season', action='store', help='Season in year', dest="year_season", required=True)
#parser.add_argument('-n', '--next-season', action='store', help='Season in year', dest="next_year_season", required=True)
#args = parser.parse_args()
#
#season = args.year_season
#next_season = args.next_year_season

#data_directory = "./nba_data"
team_stat_path = './nba_data/*.csv'
team_stat_files = glob.glob(team_stat_path)

data_types = ['Height', 'Weight', 'WL_PERC']
num_data_types = len(data_types)

def readTeamStats(file_name):
    dtypes = np.dtype({ 'names' : ('team', 'Height', 'Weight', 'WL_PERC'),
                        'formats' : ['S10', np.float, np.float, np.float] })

    data = np.loadtxt(file_name, delimiter=',', skiprows=1,
           usecols=(0,2,3,4), dtype=dtypes)

    #data_list = list(data)

    return data

def readTeamRecord(file_name):
    dtypes = np.dtype({ 'names' : ('team', 'WL_PERC'),
                        'formats' : ['S10', np.float] })

    data = np.loadtxt(file_name, delimiter=',', skiprows=1,
           usecols=(0,3), dtype=dtypes)

    return data

# Iterate through each NBA team stats file and output find the mean for each teams weight and height
def analyzeTeamStats(season):

    data_set=[]
    team_labels_set = []

    data_directory = ("/Users/aidan.wong/Documents/mystuff/cs454/jumpball/bd_collect/nba_data/%s/" % season)

    for root, dirs, files in os.walk(data_directory):
        for f in files:
            if f.endswith("agg_data.csv"):
                teamStats = readTeamStats(data_directory + f)

                teamStats_list =  zip(*teamStats)
                team = teamStats_list[0][0]
                ht_mean = np.array(teamStats_list[1], dtype=float).mean()
                wt_mean = np.array(teamStats_list[2], dtype=float).mean()
                wl_perc = teamStats_list[3][0]

                data = [ht_mean, wt_mean,wl_perc]
                data_set.append(data)
                team_labels_set.append(team)
    #print data_set
    #print "--------------"
    return data_set,team_labels_set

season = "2011"
next_year_season = "2012"
trainData_1, teamData_1 = analyzeTeamStats(season)
trainData_2, teamData_2 = analyzeTeamStats(next_year_season)
#print trainData_2
#print trainData_2[3]
print "------------------------"
print teamData_1
print teamData_2
#print teamData_2[3]

# This variable is to defined the new sample team to run K-NN with and the number should range from 0 to 30
new_team_index = 3
trainData_array = np.array(trainData_1).astype(np.float32)
newData_array = np.array(trainData_2)
#print trainData_array
#print newData_array
#print newData_array[0]

#print trainData_array

labels = np.array(np.arange(30))
#print labels
#labels = array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])

for label, x, y in zip(teamData_1,trainData_array[:,0],trainData_array[:,1]):
    pyplot.annotate(label,xy =(x,y), xytext=(-20,20),textcoords = 'offset points',
        ha ='right', va = 'bottom',bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow',
        alpha = 0.5),arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

for i in range(0,30):
    r = lambda: random.randint(0,255)
    color = '#%02X%02X%02X' % (r(),r(),r())
    team_data = trainData_array[labels.ravel()==i]
    pyplot.scatter(team_data[:,0],team_data[:,1],marker = 'o',s = team_data[:,2]*4500,c = color,cmap = pyplot.get_cmap('Spectral'))

#print newData_array[labels.ravel()==1]
#for i in range(1,2):
new_team_data = newData_array[labels.ravel()==new_team_index]
print "new_team_data =", new_team_data
pyplot.scatter(new_team_data[:,0],new_team_data[:,1],marker = '^',s = new_team_data[:,2]*4500,c = color,cmap = pyplot.get_cmap('Spectral'))

i = 0
for label, x, y in zip(teamData_2[:new_team_index+1],newData_array[:,0],newData_array[:,1]):
    if i < new_team_index:
       print i
       i += 1
       continue
    print zip(teamData_2[:new_team_index+1],newData_array[:,0],newData_array[:,1])
    print (label,x,y)
#    print type(label)
    pyplot.annotate(label,xy =(x,y), xytext=(-20,20),textcoords = 'offset points',
        ha ='right', va = 'bottom',bbox = dict(boxstyle = 'round,pad=0.5', fc = 'blue',
        alpha = 0.5),arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

knn = cv2.KNearest()
#print trainData_array
#
#
#knn.train(trainData_array,labels)
knn.train(trainData_array,np.array(labels).astype(np.float32))
ret, results, neighbours ,dist = knn.find_nearest((new_team_data).astype(np.float32), 1)
print (ret, results, neighbours, dist)
#
print "result: ", results,"\n"
print "neighbours: ", neighbours,"\n"
print "distance: ", dist

pyplot.show()


from scipy import ndimage
from collections import Counter
from core.vggnet import Vgg19
from core.utils import *

import tensorflow as tf
import numpy as np
import pandas as pd
import hickle
import os
import json

def _process_caption_data(caption_file, image_dir, max_length):
    with open(caption_file) as f:
        caption_data = json.load(f)

    # id_to_filename is a dictionary such as {image_id: filename]}
    id_to_filename = {image['id']: image['file_name'] for image in caption_data['images']}

    # data is a list of dictionary which contains 'captions', 'file_name' and 'image_id' as key.
    data = []
    for annotation in caption_data['annotations']:
        image_id = annotation['image_id']
        annotation['file_name'] = os.path.join(image_dir, id_to_filename[image_id])
        data += [annotation]

    # convert to pandas dataframe (for later visualization or debugging)
    caption_data = pd.DataFrame.from_dict(data)
    del caption_data['id']
    caption_data.sort_values(by='image_id', inplace=True)
    caption_data = caption_data.reset_index(drop=True)

    del_idx = []
    for i, caption in enumerate(caption_data['caption']):
        caption = caption.replace('.','').replace(',','').replace("'","").replace('"','')
        caption = caption.replace('&','and').replace('(','').replace(")","").replace('-',' ')
        caption = " ".join(caption.split())  # replace multiple spaces

        caption_data.set_value(i, 'caption', caption.lower())
        if len(caption.split(" ")) > max_length:
            del_idx.append(i)

    # delete captions if size is larger than max_length
    print "The number of captions before deletion: %d" %len(caption_data)
    caption_data = caption_data.drop(caption_data.index[del_idx])
    caption_data = caption_data.reset_index(drop=True)
    print "The number of captions after deletion: %d" %len(caption_data)
    return caption_data

def _build_vocab(annotations, threshold=1):
    counter = Counter()
    max_len = 0
    for i, caption in enumerate(annotations['caption']):
        words = caption.split(' ') # caption contrains only lower-case words
        for w in words:
            counter[w] +=1

        if len(caption.split(" ")) > max_len:
            max_len = len(caption.split(" "))

    vocab = [word for word in counter if counter[word] >= threshold]
    print ('Filtered %d words to %d words with word count threshold %d.' % (len(counter), len(vocab), threshold))

    word_to_idx = {u'<NULL>': 0, u'<START>': 1, u'<END>': 2}
    idx = 3
    for word in vocab:
        word_to_idx[word] = idx
        idx += 1
    print "Max length of caption: ", max_len
    return word_to_idx

def _build_caption_vector(annotations, word_to_idx, max_length=15):
    n_examples = len(annotations)
    captions = np.ndarray((n_examples,max_length+2)).astype(np.int32)

    for i, caption in enumerate(annotations['caption']):
        words = caption.split(" ") # caption contrains only lower-case words
        cap_vec = []
        cap_vec.append(word_to_idx['<START>'])
        for word in words:
            if word in word_to_idx:
                cap_vec.append(word_to_idx[word])
        cap_vec.append(word_to_idx['<END>'])

        # pad short caption with the special null token '<NULL>' to make it fixed-size vector
        if len(cap_vec) < (max_length + 2):
            for j in range(max_length + 2 - len(cap_vec)):
                cap_vec.append(word_to_idx['<NULL>'])

        captions[i, :] = np.asarray(cap_vec)
    print "Finished building caption vectors"
    return captions

def _build_file_names(annotations):
    image_file_names = []
    id_to_idx = {}
    idx = 0
    image_ids = annotations['image_id']
    file_names = annotations['file_name']
    for image_id, file_name in zip(image_ids, file_names):
        if not image_id in id_to_idx:
            id_to_idx[image_id] = idx
            image_file_names.append(file_name)
            idx += 1

    file_names = np.asarray(image_file_names)
    return file_names, id_to_idx

def _build_image_idxs(annotations, id_to_idx):
    image_idxs = np.ndarray(len(annotations), dtype=np.int32)
    image_ids = annotations['image_id']
    for i, image_id in enumerate(image_ids):
        image_idxs[i] = id_to_idx[image_id]
    return image_idxs

def main():
    # batch size for extracting feature vectors from vggnet.
    batch_size = 100
    # maximum length of caption(number of word). if caption is longer than max_length, deleted.
    max_length = 15
    # if word occurs less than word_count_threshold in training dataset, the word index is special unknown token.
    word_count_threshold = 1
    # vgg model path
    vgg_model_path = './data/imagenet-vgg-verydeep-19.mat'

    caption_file = 'data/annotations/captions_train2014.json'
    image_dir = 'image/%2014_resized/'

    # about 80000 images and 400000 captions for train dataset
    train_dataset = _process_caption_data(caption_file='data/annotations/captions_train2014.json',
                                          image_dir='image/train2014_resized/',
                                          max_length=max_length)

    # about 40000 images and 200000 captions
    val_dataset = _process_caption_data(caption_file='data/annotations/captions_val2014.json',
                                        image_dir='image/val2014_resized/',
                                        max_length=max_length)

    # about 4000 images and 20000 captions for val / test dataset
    val_cutoff = int(0.1 * len(val_dataset))
    test_cutoff = int(0.2 * len(val_dataset))
    print 'Finished processing caption data'

    save_pickle(train_dataset, 'data/train/train.annotations.pkl')
    save_pickle(val_dataset[:val_cutoff], 'data/val/val.annotations.pkl')
    save_pickle(val_dataset[val_cutoff:test_cutoff].reset_index(drop=True), 'data/test/test.annotations.pkl')

    for split in ['train', 'val', 'test']:
        annotations = load_pickle('./data/%s/%s.annotations.pkl' % (split, split))

        if split == 'train':
            word_to_idx = _build_vocab(annotations=annotations, threshold=word_count_threshold)
            save_pickle(word_to_idx, './data/%s/word_to_idx.pkl' % split)

        captions = _build_caption_vector(annotations=annotations, word_to_idx=word_to_idx, max_length=max_length)
        save_pickle(captions, './data/%s/%s.captions.pkl' % (split, split))

        file_names, id_to_idx = _build_file_names(annotations)
        save_pickle(file_names, './data/%s/%s.file.names.pkl' % (split, split))

        image_idxs = _build_image_idxs(annotations, id_to_idx)
        save_pickle(image_idxs, './data/%s/%s.image.idxs.pkl' % (split, split))

        # prepare reference captions to compute bleu scores later
        image_ids = {}
        feature_to_captions = {}
        i = -1
        for caption, image_id in zip(annotations['caption'], annotations['image_id']):
            if not image_id in image_ids:
                image_ids[image_id] = 0
                i += 1
                feature_to_captions[i] = []
            feature_to_captions[i].append(caption.lower() + ' .')
        save_pickle(feature_to_captions, './data/%s/%s.references.pkl' % (split, split))
        print "Finished building %s caption dataset" %split

    # extract conv5_3 feature vectors
    vggnet = Vgg19(vgg_model_path)
    vggnet.build()
    with tf.Session() as sess:
        tf.initialize_all_variables().run()
        for split in ['train', 'val', 'test']:
            anno_path = './data/%s/%s.annotations.pkl' % (split, split)
            save_path = './data/%s/%s.features.hkl' % (split, split)
            annotations = load_pickle(anno_path)
            image_path = list(annotations['file_name'].unique())
            n_examples = len(image_path)

            all_feats = np.ndarray([n_examples, 196, 512], dtype=np.float32)

            for start, end in zip(range(0, n_examples, batch_size),
                                  range(batch_size, n_examples + batch_size, batch_size)):
                image_batch_file = image_path[start:end]
                image_batch = np.array(map(lambda x: ndimage.imread(x, mode='RGB'), image_batch_file)).astype(
                    np.float32)
                feats = sess.run(vggnet.features, feed_dict={vggnet.images: image_batch})
                all_feats[start:end, :] = feats
                print ("Processed %d %s features.." % (end, split))

            # use hickle to save huge feature vectors
            hickle.dump(all_feats, save_path)
            print ("Saved %s.." % (save_path))

if __name__ == "__main__":
    main()


from __future__ import division

import numpy as np

from sklearn.utils.linear_assignment_ import linear_assignment
from sklearn.utils.validation import check_consistent_length, check_array

__all__ = ["consensus_score"]

def _check_rows_and_columns(a, b):
    """Unpacks the row and column arrays and checks their shape."""
    check_consistent_length(*a)
    check_consistent_length(*b)
    checks = lambda x: check_array(x, ensure_2d=False)
    a_rows, a_cols = map(checks, a)
    b_rows, b_cols = map(checks, b)
    return a_rows, a_cols, b_rows, b_cols

def _jaccard(a_rows, a_cols, b_rows, b_cols):
    """Jaccard coefficient on the elements of the two biclusters."""
    intersection = ((a_rows * b_rows).sum() *
                    (a_cols * b_cols).sum())

    a_size = a_rows.sum() * a_cols.sum()
    b_size = b_rows.sum() * b_cols.sum()

    return intersection / (a_size + b_size - intersection)

def _pairwise_similarity(a, b, similarity):
    """Computes pairwise similarity matrix.

    result[i, j] is the Jaccard coefficient of a's bicluster i and b's
    bicluster j.

    """
    a_rows, a_cols, b_rows, b_cols = _check_rows_and_columns(a, b)
    n_a = a_rows.shape[0]
    n_b = b_rows.shape[0]
    result = np.array(list(list(similarity(a_rows[i], a_cols[i],
                                           b_rows[j], b_cols[j])
                                for j in range(n_b))
                           for i in range(n_a)))
    return result

def consensus_score(a, b, similarity="jaccard"):
    """The similarity of two sets of biclusters.

    Similarity between individual biclusters is computed. Then the
    best matching between sets is found using the Hungarian algorithm.
    The final score is the sum of similarities divided by the size of
    the larger set.

    Read more in the :ref:`User Guide <biclustering>`.

    Parameters
    ----------
    a : (rows, columns)
        Tuple of row and column indicators for a set of biclusters.

    b : (rows, columns)
        Another set of biclusters like ``a``.

    similarity : string or function, optional, default: "jaccard"
        May be the string "jaccard" to use the Jaccard coefficient, or
        any function that takes four arguments, each of which is a 1d
        indicator vector: (a_rows, a_columns, b_rows, b_columns).

    References
    ----------

    * Hochreiter, Bodenhofer, et. al., 2010. `FABIA: factor analysis
      for bicluster acquisition
      <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2881408/>`__.

    """
    if similarity == "jaccard":
        similarity = _jaccard
    matrix = _pairwise_similarity(a, b, similarity)
    indices = linear_assignment(1. - matrix)
    n_a = len(a[0])
    n_b = len(b[0])
    return matrix[indices[:, 0], indices[:, 1]].sum() / max(n_a, n_b)


#  Copyright (c) 2021, Manfred Moitzi
#  License: MIT License
import sys
from typing import Iterable
import pathlib
import random
import ezdxf
from ezdxf import zoom, print_config
from ezdxf.math import Matrix44
from ezdxf.tools import fonts
from ezdxf.tools import text_layout as tl

"""
This example shows the usage of the internal text_layout module to render
complex text layouts. The module is designed to render MText like entities,
but could be used for other tasks too. The layout engine supports a multi
column setup, each column contains paragraphs, and these paragraphs can
automatically flow across the columns. All locations are relative to each other,
absolute locations are not supported - tabulators are not supported.

The layout engine knows nothing about the content itself, it just manages
content boxes of a fixed given width and height and "glue" spaces in between.
The engine does not alter the size of the content boxes, but resizes the glue
if necessary. The actual rendering is done by a rendering object associated to
each content box.

The only text styling manged by the layout engine is underline, overline and
strike through multiple content boxes.

Features:

- layout alignment like MText: top-middle-bottom combined with left-center-right
- paragraph alignments: left, right, center, justified
- paragraph indentation: left, right, special first line
- cell alignments: top, center, bottom
- fraction cells: over, slanted, tolerance style
- columns have a fixed height or grows automatically, paragraphs which do not
  fit "flow" into the following column.
- pass through of transformation matrix to the rendering object

TODO:

- bullet- and numbered lists
- refinements to replicate MText features as good as possible

Used for:

- drawing add-on to render MTEXT with columns
- explode MTEXT into DXF primitives (TEXT, LINE)

"""
if not ezdxf.options.use_matplotlib:
    print("The Matplotlib package is required.")
    sys.exit(1)

# Type alias:
Content = Iterable[tl.Cell]

DIR = pathlib.Path("~/Desktop/Outbox").expanduser()
STYLE = "Style0"
FONT = "OpenSans-Regular.ttf"
COLUMN_HEIGHT: float = 12

print_config()

doc = ezdxf.new()
msp = doc.modelspace()
style = doc.styles.new(STYLE, dxfattribs={"font": FONT})

def measure_space(font):
    return font.text_width(" X") - font.text_width("X")

class SizedFont:
    def __init__(self, height: float):
        self.height = float(height)
        self.font = fonts.make_font(FONT, self.height)
        self.space = measure_space(self.font)

    def text_width(self, text: str):
        return self.font.text_width(text)

fix_sized_fonts = [
    SizedFont(0.18),
    SizedFont(0.35),
    SizedFont(0.50),
    SizedFont(0.70),
    SizedFont(1.00),
]

class FrameRenderer(tl.ContentRenderer):
    """Render object to render a frame around a content collection.

    This renderer can be used by collections which just manages content
    but do not represent a content by itself (Layout, Column, Paragraph).

    """

    def __init__(self, color):
        self.color = color

    def render(
        self,
        left: float,
        bottom: float,
        right: float,
        top: float,
        m: Matrix44 = None,
    ) -> None:
        """Render a frame as LWPOLYLINE."""
        pline = msp.add_lwpolyline(
            [(left, top), (right, top), (right, bottom), (left, bottom)],
            close=True,
            dxfattribs={"color": self.color},
        )
        if m:
            pline.transform(m)

    def line(
        self, x1: float, y1: float, x2: float, y2: float, m: Matrix44 = None
    ) -> None:
        """Line renderer used to create underline, overline, strike through
        and fraction dividers.

        """
        line = msp.add_line(
            (x1, y1), (x2, y2), dxfattribs={"color": self.color}
        )
        if m:
            line.transform(m)

class TextRenderer(tl.ContentRenderer):
    """Text content renderer."""

    def __init__(self, text, attribs):
        self.text = text
        self.attribs = attribs
        self.line_attribs = {"color": attribs["color"]}

    def render(
        self,
        left: float,
        bottom: float,
        right: float,
        top: float,
        m: Matrix44 = None,
    ):
        """Create/render the text content"""
        text = msp.add_text(self.text, dxfattribs=self.attribs)
        text.set_pos((left, bottom), align="LEFT")
        if m:
            text.transform(m)

    def line(
        self, x1: float, y1: float, x2: float, y2: float, m: Matrix44 = None
    ) -> None:
        """Line renderer used to create underline, overline, strike through
        and fraction dividers.

        """
        line = msp.add_line((x1, y1), (x2, y2), dxfattribs=self.line_attribs)
        if m:
            line.transform(m)

class Word(tl.Text):
    """Represent a word as content box for the layout engine."""

    def __init__(self, text: str, font: SizedFont, stroke: int = 0):
        # Each content box can have individual properties:
        attribs = {
            "color": random.choice((1, 2, 3, 4, 6, 7, 7)),
            "height": font.height,
            "style": STYLE,
        }
        super().__init__(
            # Width and height of the content are fixed given values and will
            # not be changed by the layout engine:
            width=font.text_width(text),
            height=font.height,
            stroke=stroke,
            # Each content box can have it's own rendering object:
            renderer=TextRenderer(text, attribs),
        )

def uniform_content(count: int, size: int = 1) -> Content:
    """Create content with one text size."""
    font = fix_sized_fonts[size]
    for word in tl.lorem_ipsum(count):
        yield Word(word, font)
        yield tl.Space(font.space)

def random_sized_content(count: int) -> Content:
    """Create content with randomized text size."""

    def size():
        return random.choice([0, 1, 1, 1, 1, 1, 2, 3])

    for word in tl.lorem_ipsum(count):
        font = fix_sized_fonts[size()]
        yield Word(word, font)
        yield tl.Space(font.space)

def stroke_groups(words: Iterable[str]):
    group = []
    count = 0
    stroke = 0
    for word in words:
        if count == 0:
            if group:
                yield group, stroke
            count = random.randint(1, 4)
            group = [word]
            stroke = random.choice([0, 0, 0, 0, 1, 1, 1, 2, 2, 4])
        else:
            count -= 1
            group.append(word)

    if group:
        yield group, stroke

def stroked_content(count: int, size: int = 1) -> Content:
    """Create content with one text size and groups of words with or without
    strokes.

    """
    font = fix_sized_fonts[size]
    groups = stroke_groups(tl.lorem_ipsum(count))
    for group, stroke in groups:
        # strokes should span across spaces in between words:
        # Spaces between words are bound to the preceding content box renderer,
        # MText is more flexible, but this implementation is easy and good
        # enough, otherwise spaces would need a distinct height and a rendering
        # object, both are not implemented for glue objects.
        continue_stroke = stroke + 8 if stroke else 0
        for word in group[:-1]:
            yield Word(word, font=font, stroke=continue_stroke)
            yield tl.Space(font.space)
        # strokes end at the last word, without continue stroke:
        yield Word(group[-1], font=font, stroke=stroke)
        yield tl.Space(font.space)

class Fraction(tl.Fraction):
    """Represents a fraction for the layout engine, which consist of a top-
    and bottom content box, divided by horizontal or slanted line.
    The "tolerance style" has no line between the stacked content boxes.

    This implementation is more flexible than MText, the content boxes can be
    words but also fractions or cell groups.

    """

    def __init__(
        self, t1: str, t2: str, stacking: tl.Stacking, font: SizedFont
    ):
        top = Word(t1, font)
        bottom = Word(t2, font)
        super().__init__(
            top=top,
            bottom=bottom,
            stacking=stacking,
            # Uses only the generic line renderer to render the divider line,
            # the top- and bottom content boxes use their own render objects.
            renderer=FrameRenderer(color=7),
        )

def fraction_content() -> Content:
    """Create content with one text size and place random fractions between
    words.

    """
    words = list(uniform_content(120))
    for word in words:
        word.valign = tl.CellAlignment.BOTTOM

    stacking_options = list(tl.Stacking)
    font = SizedFont(0.25)  # fraction font
    for _ in range(10):
        stacking = random.choice(stacking_options)
        top = str(random.randint(1, 1000))
        bottom = str(random.randint(1, 1000))
        pos = random.randint(0, len(words) - 1)
        if isinstance(words[pos], tl.Space):
            pos += 1
        words.insert(pos, Fraction(top, bottom, stacking, font))
        words.insert(pos + 1, tl.Space(font.space))
    return words

def create_layout(align: tl.ParagraphAlignment, content: Content) -> tl.Layout:
    # Create a flow text paragraph for the content:
    paragraph = tl.Paragraph(align=align)
    paragraph.append_content(content)

    # Start the layout engine and set default column width:
    layout = tl.Layout(
        width=8,  # default column width for columns without define width
        margins=(0.5,),  # space around the layout
        # The render object of collections like Layout, Column or Paragraph is
        # called before the render objects of the content managed by the
        # collection.
        # This could be used to render a frame or a background:
        renderer=FrameRenderer(color=2),
    )

    # Append the first column with default width and a content height of 12 drawing
    # units. At least the first column has to be created by the client.
    layout.append_column(height=COLUMN_HEIGHT, gutter=1)

    # Append the content. The content will be distributed across the available
    # columns and automatically overflow into adjacent columns if necessary.
    # The layout engine creates new columns automatically if required by
    # cloning the last column.
    layout.append_paragraphs([paragraph])

    # Content- and total size is always up to date, only the final location
    # has to be updated by calling Layout.place().
    print()
    print(f"Layout has {len(layout)} columns.")
    print(f"Layout total width: {layout.total_width}")
    print(f"Layout total height: {layout.total_height}")

    for n, column in enumerate(layout, start=1):
        print()
        print(f"  {n}. column has {len(column)} paragraph(s)")
        print(f"  Column total width: {column.total_width}")
        print(f"  Column total height: {column.total_height}")

    # It is recommended to place the layout at origin (0, 0) and use a
    # transformation matrix to move the layout to the final location in
    # the DXF target layout - the model space in this example.
    # Set final layout location in the xy-plane with alignment:
    layout.place(align=tl.LayoutAlignment.BOTTOM_LEFT)

    # It is possible to add content after calling place(), but place has to be
    # called again before calling the render() method of the layout.
    return layout

def create(content: Content, y: float) -> None:
    x: float = 0
    for align in list(tl.ParagraphAlignment):
        # Build and place the layout at (0, 0):
        layout = create_layout(align, content)
        # Render and move the layout to the final location:
        m = Matrix44.translate(x, y, 0)
        layout.render(m)
        x += layout.total_width + 2

dy: float = COLUMN_HEIGHT + 3
create(list(uniform_content(200)), 0)
create(list(random_sized_content(200)), dy)
create(list(stroked_content(200)), 2 * dy)
create(fraction_content(), 3 * dy)

# zooming needs the longest time:
zoom.extents(msp, factor=1.1)
doc.saveas(str(DIR / "text_layout.dxf"))


# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Linear regression using the LinearRegressor Estimator."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

import imports85  # pylint: disable=g-bad-import-order

STEPS = 1000

def main(argv):
  """Builds, trains, and evaluates the model."""
  assert len(argv) == 1
  (x_train, y_train), (x_test, y_test) = imports85.load_data()

  # Build the training input_fn.
  input_train = tf.estimator.inputs.pandas_input_fn(
      x=x_train,
      y=y_train,
      # Setting `num_epochs` to `None` lets the `inpuf_fn` generate data
      # indefinitely, leaving the call to `Estimator.train` in control.
      num_epochs=None,
      shuffle=True)

  # Build the validation input_fn.
  input_test = tf.estimator.inputs.pandas_input_fn(
      x=x_test, y=y_test, shuffle=True)

  feature_columns = [
      # "curb-weight" and "highway-mpg" are numeric columns.
      tf.feature_column.numeric_column(key="curb-weight"),
      tf.feature_column.numeric_column(key="highway-mpg"),
  ]

  # Build the Estimator.
  model = tf.estimator.LinearRegressor(feature_columns=feature_columns)

  # Train the model.
  # By default, the Estimators log output every 100 steps.
  model.train(input_fn=input_train, steps=STEPS)

  # Evaluate how the model performs on data it has not yet seen.
  eval_result = model.evaluate(input_fn=input_test)

  # The evaluation returns a Python dictionary. The "average_loss" key holds the
  # Mean Squared Error (MSE).
  average_loss = eval_result["average_loss"]

  # Convert MSE to Root Mean Square Error (RMSE).
  print("\n" + 80 * "*")
  print("\nRMS error for the test set: ${:.0f}".format(average_loss**0.5))

  # Run the model in prediction mode.
  input_dict = {
      "curb-weight": np.array([2000, 3000]),
      "highway-mpg": np.array([30, 40])
  }
  predict_input_fn = tf.estimator.inputs.numpy_input_fn(
      input_dict, shuffle=False)
  predict_results = model.predict(input_fn=predict_input_fn)

  # Print the prediction results.
  print("\nPrediction results:")
  for i, prediction in enumerate(predict_results):
    msg = ("Curb weight: {: 4d}lbs, "
           "Highway: {: 0d}mpg, "
           "Prediction: ${: 9.2f}")
    msg = msg.format(input_dict["curb-weight"][i], input_dict["highway-mpg"][i],
                     prediction["predictions"][0])

    print("    " + msg)
  print()

if __name__ == "__main__":
  # The Estimator periodically generates "INFO" logs; make these logs visible.
  tf.logging.set_verbosity(tf.logging.INFO)
  tf.app.run(main=main)


"""
==================================================
Automatic Relevance Determination Regression (ARD)
==================================================

Fit regression model with Bayesian Ridge Regression.

See :ref:`bayesian_ridge_regression` for more information on the regressor.

Compared to the OLS (ordinary least squares) estimator, the coefficient
weights are slightly shifted toward zeros, which stabilises them.

The histogram of the estimated weights is very peaked, as a sparsity-inducing
prior is implied on the weights.

The estimation of the model is done by iteratively maximizing the
marginal log-likelihood of the observations.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.linear_model import ARDRegression, LinearRegression

###############################################################################
# Generating simulated data with Gaussian weights

# Parameters of the example
np.random.seed(0)
n_samples, n_features = 100, 100
# Create Gaussian data
X = np.random.randn(n_samples, n_features)
# Create weigts with a precision lambda_ of 4.
lambda_ = 4.
w = np.zeros(n_features)
# Only keep 10 weights of interest
relevant_features = np.random.randint(0, n_features, 10)
for i in relevant_features:
    w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_))
# Create noite with a precision alpha of 50.
alpha_ = 50.
noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples)
# Create the target
y = np.dot(X, w) + noise

###############################################################################
# Fit the ARD Regression
clf = ARDRegression(compute_score=True)
clf.fit(X, y)

ols = LinearRegression()
ols.fit(X, y)

###############################################################################
# Plot the true weights, the estimated weights and the histogram of the
# weights
plt.figure(figsize=(6, 5))
plt.title("Weights of the model")
plt.plot(clf.coef_, 'b-', label="ARD estimate")
plt.plot(ols.coef_, 'r--', label="OLS estimate")
plt.plot(w, 'g-', label="Ground truth")
plt.xlabel("Features")
plt.ylabel("Values of the weights")
plt.legend(loc=1)

plt.figure(figsize=(6, 5))
plt.title("Histogram of the weights")
plt.hist(clf.coef_, bins=n_features, log=True)
plt.plot(clf.coef_[relevant_features], 5 * np.ones(len(relevant_features)),
         'ro', label="Relevant features")
plt.ylabel("Features")
plt.xlabel("Values of the weights")
plt.legend(loc=1)

plt.figure(figsize=(6, 5))
plt.title("Marginal log-likelihood")
plt.plot(clf.scores_)
plt.ylabel("Score")
plt.xlabel("Iterations")
plt.show()


#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import unittest

from pyspark.sql.functions import array, explode, col, lit, udf, pandas_udf
from pyspark.sql.types import DoubleType, StructType, StructField, Row
from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
    pandas_requirement_message, pyarrow_requirement_message
from pyspark.testing.utils import QuietTest

if have_pandas:
    import pandas as pd
    from pandas.testing import assert_frame_equal

if have_pyarrow:
    import pyarrow as pa  # noqa: F401

@unittest.skipIf(
    not have_pandas or not have_pyarrow,
    pandas_requirement_message or pyarrow_requirement_message)  # type: ignore[arg-type]
class CogroupedMapInPandasTests(ReusedSQLTestCase):

    @property
    def data1(self):
        return self.spark.range(10).toDF('id') \
            .withColumn("ks", array([lit(i) for i in range(20, 30)])) \
            .withColumn("k", explode(col('ks')))\
            .withColumn("v", col('k') * 10)\
            .drop('ks')

    @property
    def data2(self):
        return self.spark.range(10).toDF('id') \
            .withColumn("ks", array([lit(i) for i in range(20, 30)])) \
            .withColumn("k", explode(col('ks'))) \
            .withColumn("v2", col('k') * 100) \
            .drop('ks')

    def test_simple(self):
        self._test_merge(self.data1, self.data2)

    def test_left_group_empty(self):
        left = self.data1.where(col("id") % 2 == 0)
        self._test_merge(left, self.data2)

    def test_right_group_empty(self):
        right = self.data2.where(col("id") % 2 == 0)
        self._test_merge(self.data1, right)

    def test_different_schemas(self):
        right = self.data2.withColumn('v3', lit('a'))
        self._test_merge(self.data1, right, 'id long, k int, v int, v2 int, v3 string')

    def test_complex_group_by(self):
        left = pd.DataFrame.from_dict({
            'id': [1, 2, 3],
            'k':  [5, 6, 7],
            'v': [9, 10, 11]
        })

        right = pd.DataFrame.from_dict({
            'id': [11, 12, 13],
            'k': [5, 6, 7],
            'v2': [90, 100, 110]
        })

        left_gdf = self.spark\
            .createDataFrame(left)\
            .groupby(col('id') % 2 == 0)

        right_gdf = self.spark \
            .createDataFrame(right) \
            .groupby(col('id') % 2 == 0)

        def merge_pandas(l, r):
            return pd.merge(l[['k', 'v']], r[['k', 'v2']], on=['k'])

        result = left_gdf \
            .cogroup(right_gdf) \
            .applyInPandas(merge_pandas, 'k long, v long, v2 long') \
            .sort(['k']) \
            .toPandas()

        expected = pd.DataFrame.from_dict({
            'k': [5, 6, 7],
            'v': [9, 10, 11],
            'v2': [90, 100, 110]
        })

        assert_frame_equal(expected, result)

    def test_empty_group_by(self):
        left = self.data1
        right = self.data2

        def merge_pandas(l, r):
            return pd.merge(l, r, on=['id', 'k'])

        result = left.groupby().cogroup(right.groupby())\
            .applyInPandas(merge_pandas, 'id long, k int, v int, v2 int') \
            .sort(['id', 'k']) \
            .toPandas()

        left = left.toPandas()
        right = right.toPandas()

        expected = pd \
            .merge(left, right, on=['id', 'k']) \
            .sort_values(by=['id', 'k'])

        assert_frame_equal(expected, result)

    def test_mixed_scalar_udfs_followed_by_cogrouby_apply(self):
        df = self.spark.range(0, 10).toDF('v1')
        df = df.withColumn('v2', udf(lambda x: x + 1, 'int')(df['v1'])) \
            .withColumn('v3', pandas_udf(lambda x: x + 2, 'int')(df['v1']))

        result = df.groupby().cogroup(df.groupby()) \
            .applyInPandas(lambda x, y: pd.DataFrame([(x.sum().sum(), y.sum().sum())]),
                           'sum1 int, sum2 int').collect()

        self.assertEqual(result[0]['sum1'], 165)
        self.assertEqual(result[0]['sum2'], 165)

    def test_with_key_left(self):
        self._test_with_key(self.data1, self.data1, isLeft=True)

    def test_with_key_right(self):
        self._test_with_key(self.data1, self.data1, isLeft=False)

    def test_with_key_left_group_empty(self):
        left = self.data1.where(col("id") % 2 == 0)
        self._test_with_key(left, self.data1, isLeft=True)

    def test_with_key_right_group_empty(self):
        right = self.data1.where(col("id") % 2 == 0)
        self._test_with_key(self.data1, right, isLeft=False)

    def test_with_key_complex(self):

        def left_assign_key(key, l, _):
            return l.assign(key=key[0])

        result = self.data1 \
            .groupby(col('id') % 2 == 0)\
            .cogroup(self.data2.groupby(col('id') % 2 == 0)) \
            .applyInPandas(left_assign_key, 'id long, k int, v int, key boolean') \
            .sort(['id', 'k']) \
            .toPandas()

        expected = self.data1.toPandas()
        expected = expected.assign(key=expected.id % 2 == 0)

        assert_frame_equal(expected, result)

    def test_wrong_return_type(self):
        # Test that we get a sensible exception invalid values passed to apply
        left = self.data1
        right = self.data2
        with QuietTest(self.sc):
            with self.assertRaisesRegex(
                    NotImplementedError,
                    'Invalid return type.*ArrayType.*TimestampType'):
                left.groupby('id').cogroup(right.groupby('id')).applyInPandas(
                    lambda l, r: l, 'id long, v array<timestamp>')

    def test_wrong_args(self):
        left = self.data1
        right = self.data2
        with self.assertRaisesRegex(ValueError, 'Invalid function'):
            left.groupby('id').cogroup(right.groupby('id')) \
                .applyInPandas(lambda: 1, StructType([StructField("d", DoubleType())]))

    def test_case_insensitive_grouping_column(self):
        # SPARK-31915: case-insensitive grouping column should work.
        df1 = self.spark.createDataFrame([(1, 1)], ("column", "value"))

        row = df1.groupby("ColUmn").cogroup(
            df1.groupby("COLUMN")
        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
        self.assertEqual(row.asDict(), Row(column=2, value=2).asDict())

        df2 = self.spark.createDataFrame([(1, 1)], ("column", "value"))

        row = df1.groupby("ColUmn").cogroup(
            df2.groupby("COLUMN")
        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
        self.assertEqual(row.asDict(), Row(column=2, value=2).asDict())

    def test_self_join(self):
        # SPARK-34319: self-join with FlatMapCoGroupsInPandas
        df = self.spark.createDataFrame([(1, 1)], ("column", "value"))

        row = df.groupby("ColUmn").cogroup(
            df.groupby("COLUMN")
        ).applyInPandas(lambda r, l: r + l, "column long, value long")

        row = row.join(row).first()

        self.assertEqual(row.asDict(), Row(column=2, value=2).asDict())

    @staticmethod
    def _test_with_key(left, right, isLeft):

        def right_assign_key(key, l, r):
            return l.assign(key=key[0]) if isLeft else r.assign(key=key[0])

        result = left \
            .groupby('id') \
            .cogroup(right.groupby('id')) \
            .applyInPandas(right_assign_key, 'id long, k int, v int, key long') \
            .toPandas()

        expected = left.toPandas() if isLeft else right.toPandas()
        expected = expected.assign(key=expected.id)

        assert_frame_equal(expected, result)

    @staticmethod
    def _test_merge(left, right, output_schema='id long, k int, v int, v2 int'):

        def merge_pandas(l, r):
            return pd.merge(l, r, on=['id', 'k'])

        result = left \
            .groupby('id') \
            .cogroup(right.groupby('id')) \
            .applyInPandas(merge_pandas, output_schema)\
            .sort(['id', 'k']) \
            .toPandas()

        left = left.toPandas()
        right = right.toPandas()

        expected = pd \
            .merge(left, right, on=['id', 'k']) \
            .sort_values(by=['id', 'k'])

        assert_frame_equal(expected, result)

if __name__ == "__main__":
    from pyspark.sql.tests.test_pandas_cogrouped_map import *  # noqa: F401

    try:
        import xmlrunner  # type: ignore[import]
        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
    except ImportError:
        testRunner = None
    unittest.main(testRunner=testRunner, verbosity=2)


# Author: Lars Buitinck <L.J.Buitinck@uva.nl>
# License: BSD 3 clause

import numbers

import numpy as np
import scipy.sparse as sp

from . import _hashing
from ..base import BaseEstimator, TransformerMixin

def _iteritems(d):
    """Like d.iteritems, but accepts any collections.Mapping."""
    return d.iteritems() if hasattr(d, "iteritems") else d.items()

class FeatureHasher(BaseEstimator, TransformerMixin):
    """Implements feature hashing, aka the hashing trick.

    This class turns sequences of symbolic feature names (strings) into
    scipy.sparse matrices, using a hash function to compute the matrix column
    corresponding to a name. The hash function employed is the signed 32-bit
    version of Murmurhash3.

    Feature names of type byte string are used as-is. Unicode strings are
    converted to UTF-8 first, but no Unicode normalization is done.
    Feature values must be (finite) numbers.

    This class is a low-memory alternative to DictVectorizer and
    CountVectorizer, intended for large-scale (online) learning and situations
    where memory is tight, e.g. when running prediction code on embedded
    devices.

    Read more in the :ref:`User Guide <feature_hashing>`.

    Parameters
    ----------
    n_features : integer, optional
        The number of features (columns) in the output matrices. Small numbers
        of features are likely to cause hash collisions, but large numbers
        will cause larger coefficient dimensions in linear learners.
    dtype : numpy type, optional
        The type of feature values. Passed to scipy.sparse matrix constructors
        as the dtype argument. Do not set this to bool, np.boolean or any
        unsigned integer type.
    input_type : string, optional
        Either "dict" (the default) to accept dictionaries over
        (feature_name, value); "pair" to accept pairs of (feature_name, value);
        or "string" to accept single strings.
        feature_name should be a string, while value should be a number.
        In the case of "string", a value of 1 is implied.
        The feature_name is hashed to find the appropriate column for the
        feature. The value's sign might be flipped in the output (but see
        non_negative, below).
    non_negative : boolean, optional, default np.float64
        Whether output matrices should contain non-negative values only;
        effectively calls abs on the matrix prior to returning it.
        When True, output values can be interpreted as frequencies.
        When False, output values will have expected value zero.

    Examples
    --------
    >>> from sklearn.feature_extraction import FeatureHasher
    >>> h = FeatureHasher(n_features=10)
    >>> D = [{'dog': 1, 'cat':2, 'elephant':4},{'dog': 2, 'run': 5}]
    >>> f = h.transform(D)
    >>> f.toarray()
    array([[ 0.,  0., -4., -1.,  0.,  0.,  0.,  0.,  0.,  2.],
           [ 0.,  0.,  0., -2., -5.,  0.,  0.,  0.,  0.,  0.]])

    See also
    --------
    DictVectorizer : vectorizes string-valued features using a hash table.
    sklearn.preprocessing.OneHotEncoder : handles nominal/categorical features
      encoded as columns of integers.
    """

    def __init__(self, n_features=(2 ** 20), input_type="dict",
                 dtype=np.float64, non_negative=False):
        self._validate_params(n_features, input_type)

        self.dtype = dtype
        self.input_type = input_type
        self.n_features = n_features
        self.non_negative = non_negative

    @staticmethod
    def _validate_params(n_features, input_type):
        # strangely, np.int16 instances are not instances of Integral,
        # while np.int64 instances are...
        if not isinstance(n_features, (numbers.Integral, np.integer)):
            raise TypeError("n_features must be integral, got %r (%s)."
                            % (n_features, type(n_features)))
        elif n_features < 1 or n_features >= 2 ** 31:
            raise ValueError("Invalid number of features (%d)." % n_features)

        if input_type not in ("dict", "pair", "string"):
            raise ValueError("input_type must be 'dict', 'pair' or 'string',"
                             " got %r." % input_type)

    def fit(self, X=None, y=None):
        """No-op.

        This method doesn't do anything. It exists purely for compatibility
        with the scikit-learn transformer API.

        Returns
        -------
        self : FeatureHasher

        """
        # repeat input validation for grid search (which calls set_params)
        self._validate_params(self.n_features, self.input_type)
        return self

    def transform(self, raw_X, y=None):
        """Transform a sequence of instances to a scipy.sparse matrix.

        Parameters
        ----------
        raw_X : iterable over iterable over raw features, length = n_samples
            Samples. Each sample must be iterable an (e.g., a list or tuple)
            containing/generating feature names (and optionally values, see
            the input_type constructor argument) which will be hashed.
            raw_X need not support the len function, so it can be the result
            of a generator; n_samples is determined on the fly.
        y : (ignored)

        Returns
        -------
        X : scipy.sparse matrix, shape = (n_samples, self.n_features)
            Feature matrix, for use with estimators or further transformers.

        """
        raw_X = iter(raw_X)
        if self.input_type == "dict":
            raw_X = (_iteritems(d) for d in raw_X)
        elif self.input_type == "string":
            raw_X = (((f, 1) for f in x) for x in raw_X)
        indices, indptr, values = \
            _hashing.transform(raw_X, self.n_features, self.dtype)
        n_samples = indptr.shape[0] - 1

        if n_samples == 0:
            raise ValueError("Cannot vectorize empty sequence.")

        X = sp.csr_matrix((values, indices, indptr), dtype=self.dtype,
                          shape=(n_samples, self.n_features))
        X.sum_duplicates()  # also sorts the indices
        if self.non_negative:
            np.abs(X.data, X.data)
        return X


# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
This file contains text pre-processing functions for NLP experiments.
"""

import os
import pandas
import re
import string

from collections import Counter
from functools import partial

class TextPreprocess(object):
  """Class for text pre-processing"""

  alphabet = string.ascii_lowercase

  def __init__(self,
               corpusTxt="compilation.txt",
               abbrCSV="abbreviations.csv",
               contrCSV="contractions.csv"):
    """
    @param corpusTxt      (str)       A compilation of most frequent words. The
        default file 'compilation.txt' is the most frequent words from both
        British National Corpus, Wiktionary, and books from Project Guttenberg.

    @param abbrCSV        (str)       A compilation of domain specific
        abbreviations. The file is a csv with the header "Abbr,Expansion". The
        default file 'abbreviations.csv' contains basic abbreviations.

    @param contrCSV       (str)       A compilation of common contractions. The
        file is a csv with the header "Contr,Expansion". The default file
        'contractions.csv' contains a short list of common contractions.
    """
    self.abbrCSV = abbrCSV
    self.contrCSV = contrCSV
    self.corpusTxt = corpusTxt

    self.abbrs = None
    self.bagOfWords = None
    self.contrs = None
    self.corpus = None

  def _setupCorpus(self, corpusSource):
    """Create member vars for English language corpus and bag of words."""
    corpusPath = os.path.abspath(os.path.join(
        os.path.dirname(__file__), '../..', 'data/etc', corpusSource))
    try:
      self.corpus = file(corpusPath).read()
    except IOError:
      raise

    self.bagOfWords = Counter(self.tokenize(self.corpus))

  def _setupAbbrs(self, abbrsSource):
    """
    Read in abbreviations, and combine all into one regex that will only match
    exact matches and not words containing them.
    E.g. if "WFH" is an abbreviation, it will match "WFH", but not "XWFH".
    """
    self.abbrs = self.readExpansionFile(abbrsSource, ["", "s", "'s"])
    self.abbrRegex = re.compile(r"\b%s\b" % r"\b|\b".join(self.abbrs.keys()))

  def _setupContr(self, contrSource):
    """
    Read in contractions, and combine all into one regex that will match any
    word ending in the contraction.
    E.g. if "'ll" is a contraction, it will match "he'll".
    """
    self.contrs = self.readExpansionFile(contrSource)
    self.contrRegex = re.compile(r"%s\b" % r"\b|".join(self.contrs.keys()))

  @staticmethod
  def readExpansionFile(filename, suffixes=None):
    """
    Read the csv file to get the original/expansion pairs and add suffixes if
    necessary.
    @param filename         (str)     Name of csv file to read. Expected format
                                      is original text in col 0, and expansion
                                      in col 1.
    @param suffixes         (list)    Strings that are added to the end of
                                      the original and expanded form if
                                      provided
    @return expansionPairs  (dict)    The keys are the original form with
                                      the suffixes added and the values are
                                      the expanded form with the suffixes
                                      added
    """
    if suffixes is None:
      suffixes = [""]

    expansionPairs = {}
    try:
      # Allow absolute paths
      if os.path.exists(filename):
        path = filename
      # Allow relative paths
      else:
        path = os.path.abspath(os.path.join(
          os.path.dirname(__file__), '../..', 'data/etc', filename))
      dataFrame = pandas.read_csv(path)

      for i in xrange(dataFrame.shape[0]):
        original = dataFrame.iloc[i][0].lower()
        expansion = dataFrame.iloc[i][1].lower()

        for suffix in suffixes:
          originalSuffix = "{}{}".format(original, suffix)
          expSuffix = "{}{}".format(expansion, suffix)
          expansionPairs[originalSuffix] = expSuffix

    except IOError:
      raise

    # Add an empty string if empty so the regex compiles
    if not expansionPairs:
      expansionPairs[""] = ""

    return expansionPairs

  def tokenize(self,
               text,
               ignoreCommon=None,
               removeStrings=None,
               correctSpell=False,
               expandAbbr=False,
               expandContr=False):
    """
    Tokenize, returning only lower-case letters and "$".
    @param text               (str)             Single string to tokenize.
    @param ignoreCommon       (int)             This many most frequent words
                                                will be filtered out from the
                                                returned tokens.
    @param removeStrings      (list)            List of strings to delete from
                                                the text.
    @param correctSpell       (bool)            Run tokens through spelling
                                                correction.
    @param expandAbbr         (bool)            Run text through abbreviation
                                                expander
    @param expandContr        (bool)            Run text through contraction
                                                expander
    """
    if not isinstance(text, str):
      raise ValueError("Must input a single string object to tokenize.")

    text = text.lower()

    if expandAbbr:
      if not self.abbrs:
        self._setupAbbrs(self.abbrCSV)
      getAbbrExpansion = partial(self.getExpansion, table=self.abbrs)
      text = self.abbrRegex.sub(getAbbrExpansion, text)

    if expandContr:
      if not self.contrs:
        self._setupContr(self.contrCSV)
      getContrExpansion = partial(self.getExpansion, table=self.contrs)
      text = self.contrRegex.sub(getContrExpansion, text)

    if removeStrings:
      for removal in removeStrings:
        text = text.replace(removal, "")

    tokens = re.findall('[a-z$]+', text)

    if correctSpell:
      tokens = [self.correct(t) for t in tokens]

    if ignoreCommon:
      tokens = self.removeMostCommon(tokens, n=ignoreCommon)

    return tokens

  def removeMostCommon(self, tokenList, n=100):
    """
    Remove the n most common tokens as counted in the bag-of-words corpus.

    @param tokenList        (list)              List of token strings.
    @param n                (int)               Will filter out the n-most
                                                frequent terms.
    """
    if not self.bagOfWords:
      self._setupCorpus(self.corpusTxt)

    ignoreList = [word[0] for word in self.bagOfWords.most_common(n)]

    return [token for token in tokenList if token not in ignoreList]

  @staticmethod
  def getExpansion(match, table):
    """
    Gets the expanded version of the regular expression
    @param match            (_sre.SRE_Match)    Regex match to expand
    @param table            (dict)              Maps the string version of the
                                                regular expression to the
                                                string expansion
    @return                 (string)            Expansion
    """
    return table[match.string[match.start(): match.end()]]

  def correct(self, word):
    """
    Find the best spelling correction for this word. Prefer edit distance  of 0,
    then one, then two; otherwise default to the word itself.
    """
    if not self.bagOfWords:
      self._setupCorpus(self.corpusTxt)

    candidates = (self._known({word}) or
                  self._known(self._editDistance1(word)) or
                  self._known(self._editDistance2(word)) or
                  [word])

    return max(candidates, key=self.bagOfWords.get)

  def _known(self, words):
    """Return the subset of words that are in the corpus."""
    return {w for w in words if w in self.bagOfWords}

  @staticmethod
  def _editDistance1(word):
    """
    Return all strings that are edit distance =1 from the input word.
    Damerau-Levenshtein edit distance:
    - deletion(x,y) is the count(xy typed as x)
    - insertion(x,y) is the count(x typed as xy)
    - substitution(x,y) is the count(x typed as y)
    - transposition(x,y) is the count(xy typed as yx)
    """
    # First split the word into tuples of all possible pairs.
    # Note: need +1 so we can perform edits at front and back end of the word.
    splits = [(word[:i], word[i:]) for i in range(len(word)+1)]

    # Now perform the edits at every possible split location.
    # Substitution is essentially a deletion and insertion.
    delete = [a+b[1:] for a,b in splits if b]
    insert = [a+b+c for a,b in splits for c in TextPreprocess.alphabet]
    subs = [a+c+b[1:] for a,b in splits for c in TextPreprocess.alphabet if b]
    trans = [a+b[1]+b[0]+b[2:] for a,b in splits if len(b)>1]

    return set(delete + insert + subs + trans)

  def _editDistance2(self, word):
    """
    Return all strings that are edit distance =2 from the input word; i.e. call
    the _editDistance1() method twice for edits with distances of two.
    """
    return {edits2 for edits1 in self._editDistance1(word)
            for edits2 in self._editDistance1(edits1)}


# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied.  See the License for the specific language governing permissions and limitations under
# the License.

"""Provides access to metric data as pandas dataframes."""

from __future__ import absolute_import

import google.cloud.monitoring

from . import _query_metadata
from . import _utils

class Query(google.cloud.monitoring.Query):
  """Query object for retrieving metric data."""

  def __init__(self,
               metric_type=google.cloud.monitoring.Query.DEFAULT_METRIC_TYPE,
               end_time=None, days=0, hours=0, minutes=0, context=None):
    """Initializes the core query parameters.

    The start time (exclusive) is determined by combining the
    values of ``days``, ``hours``, and ``minutes``, and subtracting
    the resulting duration from the end time.

    It is also allowed to omit the end time and duration here,
    in which case :meth:`~google.cloud.monitoring.query.Query.select_interval`
    must be called before the query is executed.

    Args:
      metric_type: The metric type name. The default value is
          :data:`Query.DEFAULT_METRIC_TYPE
          <google.cloud.monitoring.query.Query.DEFAULT_METRIC_TYPE>`, but
          please note that this default value is provided only for
          demonstration purposes and is subject to change.
      end_time: The end time (inclusive) of the time interval for which
          results should be returned, as a datetime object. The default
          is the start of the current minute.
      days: The number of days in the time interval.
      hours: The number of hours in the time interval.
      minutes: The number of minutes in the time interval.
      context: An optional Context object to use instead of the global default.

    Raises:
        ValueError: ``end_time`` was specified but ``days``, ``hours``, and
            ``minutes`` are all zero. If you really want to specify a point in
            time, use
            :meth:`~google.cloud.monitoring.query.Query.select_interval`.
    """
    client = _utils.make_client(context)
    super(Query, self).__init__(client, metric_type,
                                end_time=end_time,
                                days=days, hours=hours, minutes=minutes)

  def metadata(self):
    """Retrieves the metadata for the query."""
    return _query_metadata.QueryMetadata(self)


"""
===========
Simple Plot
===========

Create a simple plot.
"""
import matplotlib.pyplot as plt
import numpy as np

# nodebox section
if __name__ == '__builtin__':
    # were in nodebox
    import os
    import tempfile
    W = 800
    inset = 20
    size(W, 600)
    plt.cla()
    plt.clf()
    plt.close('all')
    def tempimage():
        fob = tempfile.NamedTemporaryFile(mode='w+b', suffix='.png', delete=False)
        fname = fob.name
        fob.close()
        return fname
    imgx = 20
    imgy = 0
    def pltshow(plt, dpi=150):
        global imgx, imgy
        temppath = tempimage()
        plt.savefig(temppath, dpi=dpi)
        dx,dy = imagesize(temppath)
        w = min(W,dx)
        image(temppath,imgx,imgy,width=w)
        imgy = imgy + dy + 20
        os.remove(temppath)
        size(W, HEIGHT+dy+40)
else:
    def pltshow(mplpyplot):
        mplpyplot.show()
# nodebox section end

# Data for plotting
t = np.arange(0.0, 2.0, 0.01)
s = 1 + np.sin(2 * np.pi * t)

# Note that using plt.subplots below is equivalent to using
# fig = plt.figure and then ax = fig.add_subplot(111)
fig, ax = plt.subplots()
ax.plot(t, s)

ax.set(xlabel='time (s)', ylabel='voltage (mV)',
       title='About as simple as it gets, folks')
ax.grid()

fig.savefig("test.png")
pltshow(plt)


import pickle
import statsmodels,statsmodels.api
import matplotlib,matplotlib.pyplot
matplotlib.rcParams.update({'font.size':36,'font.family':'Arial','xtick.labelsize':28,'ytick.labelsize':28})
thePointSize=12

# 0. user defined variables
jarDir='/Users/adriandelomana/scratch/'

# sustained trajectories
selected=['clonal.2.1','engineered.1.1','engineered.2.2','mutagenized.2.2'] # should be n = 6

# 1. iterate over selected trajectories
allx=[]; ally=[]
for replicate in selected:

    print(replicate)

    # read jar file
    jarFile=jarDir+replicate+'.pickle'
    f=open(jarFile,'rb')
    trajectory=pickle.load(f)
    f.close()

    # recover data into a format amenable to plotting
    x=trajectory[0]
    y=trajectory[1]
    z=trajectory[2]

    for a,b in zip(x,y):
        allx.append(a)
        ally.append(b)

    # run lowess over each replicate. if lowess does not work, do correlation. if not pchip

    # plot
    matplotlib.pyplot.errorbar(x,y,yerr=z,fmt='o',color='black',ecolor='black',markeredgecolor='black',capsize=0,ms=thePointSize,mew=0,alpha=0.33)

# run lowess over all data
lowess = statsmodels.api.nonparametric.lowess(ally,allx,it=10)
matplotlib.pyplot.plot(lowess[:, 0], lowess[:, 1],color='red',lw=4,zorder=100)

# close figure
matplotlib.pyplot.plot([0,300],[0,0],'--',color='black')

matplotlib.pyplot.xlim([-25,325])
matplotlib.pyplot.ylim([-0.44,0.44])
matplotlib.pyplot.xticks([0,100,200,300])
matplotlib.pyplot.yticks([-0.4,-0.2,0,0.2,0.4])
matplotlib.pyplot.xlabel('Generation')
matplotlib.pyplot.ylabel('Conditioned\nFitness')
#matplotlib.pyplot.text(-20,0.3,'Sustained')
matplotlib.pyplot.text(120,-0.38,'Sustained',color='red')

matplotlib.pyplot.tight_layout(pad=0.5)

matplotlib.pyplot.savefig('figure.sustained.pdf')


#!/usr/bin/env python
#
# Copyright 2009 Free Software Foundation, Inc.
#
# This file is part of GNU Radio
#
# GNU Radio is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GNU Radio is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNU Radio; see the file COPYING.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street,
# Boston, MA 02110-1301, USA.
#

from gnuradio import gr, blks2
import sys

try:
    import scipy
except ImportError:
    print "Error: Program requires scipy (see: www.scipy.org)."
    sys.exit(1)

try:
    import pylab
except ImportError:
    print "Error: Program requires matplotlib (see: matplotlib.sourceforge.net)."
    sys.exit(1)

class mytb(gr.top_block):
    def __init__(self, fs_in, fs_out, fc, N=10000):
        gr.top_block.__init__(self)

        rerate = float(fs_out) / float(fs_in)
        print "Resampling from %f to %f by %f " %(fs_in, fs_out, rerate)

        # Creating our own taps
        taps = gr.firdes.low_pass_2(32, 32, 0.25, 0.1, 80)

        self.src = gr.sig_source_c(fs_in, gr.GR_SIN_WAVE, fc, 1)
        #self.src = gr.noise_source_c(gr.GR_GAUSSIAN, 1)
        self.head = gr.head(gr.sizeof_gr_complex, N)

        # A resampler with our taps
        self.resamp_0 = blks2.pfb_arb_resampler_ccf(rerate, taps,
                                                    flt_size=32)

        # A resampler that just needs a resampling rate.
        # Filter is created for us and designed to cover
        # entire bandwidth of the input signal.
        # An optional atten=XX rate can be used here to
        # specify the out-of-band rejection (default=80).
        self.resamp_1 = blks2.pfb_arb_resampler_ccf(rerate)

        self.snk_in = gr.vector_sink_c()
        self.snk_0 = gr.vector_sink_c()
        self.snk_1 = gr.vector_sink_c()

        self.connect(self.src, self.head, self.snk_in)
        self.connect(self.head, self.resamp_0, self.snk_0)
        self.connect(self.head, self.resamp_1, self.snk_1)

def main():
    fs_in = 8000
    fs_out = 20000
    fc = 1000
    N = 10000

    tb = mytb(fs_in, fs_out, fc, N)
    tb.run()

    # Plot PSD of signals
    nfftsize = 2048
    fig1 = pylab.figure(1, figsize=(10,10), facecolor="w")
    sp1 = fig1.add_subplot(2,1,1)
    sp1.psd(tb.snk_in.data(), NFFT=nfftsize,
            noverlap=nfftsize/4, Fs = fs_in)
    sp1.set_title(("Input Signal at f_s=%.2f kHz" % (fs_in/1000.0)))
    sp1.set_xlim([-fs_in/2, fs_in/2])

    sp2 = fig1.add_subplot(2,1,2)
    sp2.psd(tb.snk_0.data(), NFFT=nfftsize,
            noverlap=nfftsize/4, Fs = fs_out,
            label="With our filter")
    sp2.psd(tb.snk_1.data(), NFFT=nfftsize,
            noverlap=nfftsize/4, Fs = fs_out,
            label="With auto-generated filter")
    sp2.set_title(("Output Signals at f_s=%.2f kHz" % (fs_out/1000.0)))
    sp2.set_xlim([-fs_out/2, fs_out/2])
    sp2.legend()

    # Plot signals in time
    Ts_in = 1.0/fs_in
    Ts_out = 1.0/fs_out
    t_in = scipy.arange(0, len(tb.snk_in.data())*Ts_in, Ts_in)
    t_out = scipy.arange(0, len(tb.snk_0.data())*Ts_out, Ts_out)

    fig2 = pylab.figure(2, figsize=(10,10), facecolor="w")
    sp21 = fig2.add_subplot(2,1,1)
    sp21.plot(t_in, tb.snk_in.data())
    sp21.set_title(("Input Signal at f_s=%.2f kHz" % (fs_in/1000.0)))
    sp21.set_xlim([t_in[100], t_in[200]])

    sp22 = fig2.add_subplot(2,1,2)
    sp22.plot(t_out, tb.snk_0.data(),
              label="With our filter")
    sp22.plot(t_out, tb.snk_1.data(),
              label="With auto-generated filter")
    sp22.set_title(("Output Signals at f_s=%.2f kHz" % (fs_out/1000.0)))
    r = float(fs_out)/float(fs_in)
    sp22.set_xlim([t_out[r * 100], t_out[r * 200]])
    sp22.legend()

    pylab.show()

if __name__ == "__main__":
    main()