# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""GAN for generating CJK characters.

The vast majority of this code is adapted from the pix2pix GAN described in
https://www.tensorflow.org/tutorials/generative/pix2pix. Changes include the
specific tensor dimensions, some tuning of magic numbers, and some changes to
loss functions.

TODO(ambuc): This file has type annotations because they're useful for a human
reader, but the build system doesn't yet enforce them with a strictly-typed
python build rule.
"""

import time
from typing import List, Text, Tuple

from IPython import display
import matplotlib.pyplot as plt
import tensorflow as tf

_LAMBDA = 100

def _load_image(filename: Text) -> List[List[tf.Tensor]]:
    """Given the filename of a PNG, returns a list of three tensors: a, b, a+b.

  Args:
    filename: Path to a file. The file must be a PNG and greyscale and 256x256.

  Returns:
    A list of tensors: a, b, and a+b.

  """
    image = tf.io.read_file(filename)
    image = tf.image.decode_png(image, channels=1)  # greyscale

    # Our images have a width which is divisible by three.
    w = tf.shape(image)[1] // 3

    return [
        tf.cast(image[:, n * w:(n + 1) * w, :], tf.float32) for n in range(3)
    ]

def make_datasets(files_glob: Text) -> Tuple[tf.data.Dataset, tf.data.Dataset]:
    """Makes the train/test datasets.

  Args:
    files_glob: A glob (like "/tmp/folder/*.png") of all the input images.

  Returns:
    A pair of train, test datasets of type tf.data.Dataset.

  """
    ds = tf.data.Dataset.list_files(files_glob).map(
        _load_image, num_parallel_calls=tf.data.AUTOTUNE).shuffle(400).batch(1)

    train_dataset_a = ds.shard(num_shards=3, index=0)
    train_dataset_b = ds.shard(num_shards=3, index=1)
    train_ds = train_dataset_a.concatenate(train_dataset_b)

    test_ds = ds.shard(num_shards=3, index=2)

    return train_ds, test_ds

def _downsample(filters: int,
                size: int,
                apply_batchnorm: bool = True) -> tf.keras.Sequential:
    """Downsampler from https://www.tensorflow.org/tutorials/generative/pix2pix#build_the_generator.

  Args:
    filters: The number of filters.
    size: The size of the input tensor width at this step.
    apply_batchnorm: Whether or not to apply batch normalization. Probably
      should be false on the input layer, and true elsewhere.

  Returns:
    A sequential model.
  """
    initializer = tf.random_normal_initializer(0., 0.02)
    result = tf.keras.Sequential()
    result.add(
        tf.keras.layers.Conv2D(filters,
                               size,
                               strides=2,
                               padding='same',
                               kernel_initializer=initializer,
                               use_bias=False))
    if apply_batchnorm:
        result.add(tf.keras.layers.BatchNormalization())
    result.add(tf.keras.layers.LeakyReLU())
    return result

def _upsample(filters: int,
              size: int,
              apply_dropout: bool = False) -> tf.keras.Sequential:
    """Upsampler from https://www.tensorflow.org/tutorials/generative/pix2pix#build_the_generator.

  Args:
    filters: The number of filters.
    size: The size of the input tensor width at this step.
    apply_dropout: Whether or not to apply dropout. Probably should be true for
      the first few layers and false elsewhere.

  Returns:
    A sequential model.

  """
    initializer = tf.random_normal_initializer(0., 0.02)
    result = tf.keras.Sequential()
    result.add(
        tf.keras.layers.Conv2DTranspose(filters,
                                        size,
                                        strides=2,
                                        padding='same',
                                        kernel_initializer=initializer,
                                        use_bias=False))
    result.add(tf.keras.layers.BatchNormalization())
    if apply_dropout:
        result.add(tf.keras.layers.Dropout(0.5))
    result.add(tf.keras.layers.ReLU())
    return result

def make_generator() -> tf.keras.Model:
    """Creates a generator.

  99% of this is copied directly from
  https://www.tensorflow.org/tutorials/generative/pix2pix#build_the_generator,
  except for the input shape (now two channels, two greyscale images instead of
  one RGB image) and output shape (one channel, one greyscale image instead of
  one RGB image).

  Returns:
    a tf.keras.Model which returns a 256x256x1 tensor.
  """
    inputs = tf.keras.layers.Input(shape=[256, 256, 2])

    up_stack = [
        _upsample(512, 4, apply_dropout=True),  # (bs, 2, 2, 1024)
        _upsample(512, 4, apply_dropout=True),  # (bs, 4, 4, 1024)
        _upsample(512, 4, apply_dropout=True),  # (bs, 8, 8, 1024)
        _upsample(512, 4),  # (bs, 16, 16, 1024)
        _upsample(256, 4),  # (bs, 32, 32, 512)
        _upsample(128, 4),  # (bs, 64, 64, 256)
        _upsample(64, 4),  # (bs, 128, 128, 128)
    ]

    x = inputs

    skips = []
    for down in [
            _downsample(64, 4, apply_batchnorm=False),  # (bs, 128, 128, 64)
            _downsample(128, 4),  # (bs, 64, 64, 128)
            _downsample(256, 4),  # (bs, 32, 32, 256)
            _downsample(512, 4),  # (bs, 16, 16, 512)
            _downsample(512, 4),  # (bs, 8, 8, 512)
            _downsample(512, 4),  # (bs, 4, 4, 512)
            _downsample(512, 4),  # (bs, 2, 2, 512)
            _downsample(512, 4),  # (bs, 1, 1, 512)
    ]:
        x = down(x)
        skips.append(x)

    skips = reversed(skips[:-1])

    # Upsampling and establishing the skip connections
    for up, skip in zip(up_stack, skips):
        x = up(x)
        x = tf.keras.layers.Concatenate()([x, skip])

    last = tf.keras.layers.Conv2DTranspose(
        1,  # one output channel, i.e. greyscale
        4,
        strides=2,
        padding='same',
        kernel_initializer=tf.random_normal_initializer(0., 0.02),
        activation='tanh')  # (bs, 256, 256, 3)

    x = last(x)

    return tf.keras.Model(inputs=inputs, outputs=x)

def generator_loss(loss_object: tf.keras.losses.Loss, disc_generated_output,
                   gen_output, target):
    gan_loss = loss_object(tf.ones_like(disc_generated_output),
                           disc_generated_output)
    # mean absolute error
    l1_loss = tf.reduce_mean(tf.abs(target - gen_output))
    total_gen_loss = gan_loss + (_LAMBDA * l1_loss)
    return total_gen_loss, gan_loss, l1_loss

def make_discriminator() -> tf.keras.Model:
    """Returns a discriminator.

  This is 99% the same as
  https://www.tensorflow.org/tutorials/generative/pix2pix#build_the_discriminator,
  except that the shape of the input and output tensors are different.

  Returns:
    A tf.keras.model which accepts a 256x256x2 tensor and compares it to a
    target 256x256x1 tensor.
  """
    initializer = tf.random_normal_initializer(0., 0.02)

    input_img = tf.keras.layers.Input(shape=[256, 256, 2], name='input_image')
    target_img = tf.keras.layers.Input(shape=[256, 256, 1],
                                       name='target_image')

    x = tf.keras.layers.concatenate([input_img,
                                     target_img])  # (bs, 256, 256, channels*2)

    down1 = _downsample(64, 4, False)(x)  # (bs, 128, 128, 64)
    down2 = _downsample(128, 4)(down1)  # (bs, 64, 64, 128)
    down3 = _downsample(256, 4)(down2)  # (bs, 32, 32, 256)

    zero_pad1 = tf.keras.layers.ZeroPadding2D()(down3)  # (bs, 34, 34, 256)
    conv = tf.keras.layers.Conv2D(512,
                                  4,
                                  strides=1,
                                  kernel_initializer=initializer,
                                  use_bias=False)(
                                      zero_pad1)  # (bs, 31, 31, 512)

    batchnorm1 = tf.keras.layers.BatchNormalization()(conv)

    leaky_relu = tf.keras.layers.LeakyReLU()(batchnorm1)

    zero_pad2 = tf.keras.layers.ZeroPadding2D()(
        leaky_relu)  # (bs, 33, 33, 512)

    last = tf.keras.layers.Conv2D(1,
                                  4,
                                  strides=1,
                                  kernel_initializer=initializer)(
                                      zero_pad2)  # (bs, 30, 30, 1)

    return tf.keras.Model(inputs=[input_img, target_img], outputs=last)

def discriminator_loss(loss_object: tf.keras.losses.Loss, disc_real_output,
                       disc_generated_output) -> float:
    """Returns discriminator loss.

  100% the same as
  https://www.tensorflow.org/tutorials/generative/pix2pix#build_the_discriminator.

  Args:
    loss_object: A reusable loss_object of type
      tf.keras.losses.BinaryCrossentropy.
    disc_real_output: A set of real images.
    disc_generated_output: A set of generator images.

  Returns:
    The sum of some loss functions.
  """
    real_loss = loss_object(tf.ones_like(disc_real_output), disc_real_output)

    generated_loss = loss_object(tf.zeros_like(disc_generated_output),
                                 disc_generated_output)

    return real_loss + generated_loss

def generate_images(model: tf.keras.Model, input_a: tf.Tensor,
                    input_b: tf.Tensor, target: tf.Tensor) -> None:
    """In Colab, prints [a | b | real(a,b) | predicted(a,b)] to the display.

  Args:
    model: The generator to use.
    input_a: the LHS image.
    input_b: the RHS image.
    target: The real(a,b) composition.
  """
    x = tf.concat([input_a, input_b], 3)
    x = tf.reshape(x, [256, 256, 2])
    prediction = model(x[tf.newaxis, ...], training=True)

    images = [input_a[0], input_b[0], target[0], prediction[0]]
    fig, axes = plt.subplots(1, 4)
    titles = [
        'Input Image A', 'Input Image B', 'Ground Truth', 'Predicted Image'
    ]

    for image, axis, title in zip(images, axes, titles):
        axis.set_title(title)
        axis.imshow(image[:, :, 0])
        axis.axis('off')
    fig.show()

@tf.function
def train_step(generator: tf.keras.Model,
               generator_optimizer: tf.keras.optimizers.Optimizer,
               discriminator: tf.keras.Model,
               discriminator_optimizer: tf.keras.optimizers.Optimizer,
               loss_object: tf.keras.losses.Loss, inp_a: tf.Tensor,
               inp_b: tf.Tensor, target: tf.Tensor, epoch: int,
               summary_writer: tf.summary.SummaryWriter) -> None:
    """Trains the models for one (1) epoch.

  See https://www.tensorflow.org/tutorials/generative/pix2pix#training.

  Args:
    generator: A generator model,
    generator_optimizer: and an optimizer for the generator.
    discriminator: A discriminator model,
    discriminator_optimizer: and an optimizer for the generator.
    loss_object: A reusable BinaryCrossentropy object.
    inp_a: A full-width image of the left-most component.
    inp_b: A full-width image of the right-most component.
    target: The human-authored image of the a+b character.
    epoch: The index of the epoch we're in.
    summary_writer: A SummaryWriter object for writing.... summaries.
  """
    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
        inp_x = tf.concat([inp_a, inp_b], 3)
        gen_output = generator(inp_x, training=True)

        disc_real_output = discriminator([inp_x, target], training=True)
        disc_generated_output = discriminator([inp_x, gen_output],
                                              training=True)

        gen_total_loss, gen_gan_loss, gen_l1_loss = generator_loss(
            loss_object, disc_generated_output, gen_output, target)
        disc_loss = discriminator_loss(loss_object, disc_real_output,
                                       disc_generated_output)

    # TODO(ambuc): Should this simply be gen_l1_loss?
    generator_gradients = gen_tape.gradient(gen_total_loss,
                                            generator.trainable_variables)
    discriminator_gradients = disc_tape.gradient(
        disc_loss, discriminator.trainable_variables)

    generator_optimizer.apply_gradients(
        zip(generator_gradients, generator.trainable_variables))
    discriminator_optimizer.apply_gradients(
        zip(discriminator_gradients, discriminator.trainable_variables))

    with summary_writer.as_default():
        tf.summary.scalar('gen_total_loss', gen_total_loss, step=epoch)
        tf.summary.scalar('gen_gan_loss', gen_gan_loss, step=epoch)
        tf.summary.scalar('gen_l1_loss', gen_l1_loss, step=epoch)
        tf.summary.scalar('disc_loss', disc_loss, step=epoch)

def fit(generator: tf.keras.Model,
        generator_optimizer: tf.keras.optimizers.Optimizer,
        discriminator: tf.keras.Model,
        discriminator_optimizer: tf.keras.optimizers.Optimizer,
        loss_object: tf.keras.losses.Loss, train_ds: tf.data.Dataset,
        epochs: int, test_ds: tf.data.Dataset, checkpoint: tf.train.Checkpoint,
        checkpoint_prefix: Text,
        summary_writer: tf.summary.SummaryWriter) -> None:
    """Runs for |epochs| and trains the models.

  Args:
    generator: A generator model,
    generator_optimizer: and an optimizer for the generator.
    discriminator: A discriminator model,
    discriminator_optimizer: and an optimizer for the generator.
    loss_object: A reusable BinaryCrossentropy object.
    train_ds:
    epochs: The number of epochs to train for.
    test_ds:
    checkpoint:
    checkpoint_prefix:
    summary_writer: A SummaryWriter object for writing.... summaries.
  """
    for epoch in range(epochs):
        start = time.time()

        display.clear_output(wait=True)

        for a, b, ab in test_ds.take(1):
            generate_images(generator, a, b, ab)
        print('Epoch: ', epoch)

        for n, (inp_a, inp_b, target) in train_ds.enumerate():
            print('.', end='')
            if (n + 1) % 100 == 0:
                print()
            train_step(generator, generator_optimizer, discriminator,
                       discriminator_optimizer, loss_object, inp_a, inp_b,
                       target, epoch, summary_writer)
        print()

        checkpoint.save(file_prefix=checkpoint_prefix)

        print('Time taken for epoch {} is {} sec\n'.format(
            epoch + 1,
            time.time() - start))
    checkpoint.save(file_prefix=checkpoint_prefix)


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
=========================================================
Importance of Feature Scaling
=========================================================

Feature scaling though standardization (or Z-score normalization)
can be an important preprocessing step for many machine learning
algorithms. Standardization involves rescaling the features such
that they have the properties of a standard normal distribution
with a mean of zero and a standard deviation of one.

While many algorithms (such as SVM, K-nearest neighbors, and logistic
regression) require features to be normalized, intuitively we can
think of Principle Component Analysis (PCA) as being a prime example
of when normalization is important. In PCA we are interested in the
components that maximize the variance. If one component (e.g. human
height) varies less than another (e.g. weight) because of their
respective scales (meters vs. kilos), PCA might determine that the
direction of maximal variance more closely corresponds with the
'weight' axis, if those features are not scaled. As a change in
height of one meter can be considered much more important than the
change in weight of one kilogram, this is clearly incorrect.

To illustrate this, PCA is performed comparing the use of data with
:class:`StandardScaler <sklearn.preprocessing.StandardScaler>` applied,
to unscaled data. The results are visualized and a clear difference noted.
The 1st principal component in the unscaled set can be seen. It can be seen
that feature #13 dominates the direction, being a whole two orders of
magnitude above the other features. This is contrasted when observing
the principal component for the scaled version of the data. In the scaled
version, the orders of magnitude are roughly the same across all the features.

The dataset used is the Wine Dataset available at UCI. This dataset
has continuous features that are heterogeneous in scale due to differing
properties that they measure (i.e alcohol content, and malic acid).

The transformed data is then used to train a naive Bayes classifier, and a
clear difference in prediction accuracies is observed wherein the dataset
which is scaled before PCA vastly outperforms the unscaled version.

"""
from __future__ import print_function
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.naive_bayes import GaussianNB
from sklearn import metrics
import matplotlib.pyplot as plt
from sklearn.datasets import load_wine
from sklearn.pipeline import make_pipeline
print(__doc__)

# Code source: Tyler Lanigan <tylerlanigan@gmail.com>
#              Sebastian Raschka <mail@sebastianraschka.com>

# License: BSD 3 clause

RANDOM_STATE = 42
FIG_SIZE = (10, 7)

features, target = load_wine(return_X_y=True)

# Make a train/test split using 30% test size
X_train, X_test, y_train, y_test = train_test_split(features, target,
                                                    test_size=0.30,
                                                    random_state=RANDOM_STATE)

# Fit to data and predict using pipelined GNB and PCA.
unscaled_clf = make_pipeline(PCA(n_components=2), GaussianNB())
unscaled_clf.fit(X_train, y_train)
pred_test = unscaled_clf.predict(X_test)

# Fit to data and predict using pipelined scaling, GNB and PCA.
std_clf = make_pipeline(StandardScaler(), PCA(n_components=2), GaussianNB())
std_clf.fit(X_train, y_train)
pred_test_std = std_clf.predict(X_test)

# Show prediction accuracies in scaled and unscaled data.
print('\nPrediction accuracy for the normal test dataset with PCA')
print('{:.2%}\n'.format(metrics.accuracy_score(y_test, pred_test)))

print('\nPrediction accuracy for the standardized test dataset with PCA')
print('{:.2%}\n'.format(metrics.accuracy_score(y_test, pred_test_std)))

# Extract PCA from pipeline
pca = unscaled_clf.named_steps['pca']
pca_std = std_clf.named_steps['pca']

# Show first principal componenets
print('\nPC 1 without scaling:\n', pca.components_[0])
print('\nPC 1 with scaling:\n', pca_std.components_[0])

# Scale and use PCA on X_train data for visualization.
scaler = std_clf.named_steps['standardscaler']
X_train_std = pca_std.transform(scaler.transform(X_train))

# visualize standardized vs. untouched dataset with PCA performed
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=FIG_SIZE)

for l, c, m in zip(range(0, 3), ('blue', 'red', 'green'), ('^', 's', 'o')):
    ax1.scatter(X_train[y_train == l, 0], X_train[y_train == l, 1],
                color=c,
                label='class %s' % l,
                alpha=0.5,
                marker=m
                )

for l, c, m in zip(range(0, 3), ('blue', 'red', 'green'), ('^', 's', 'o')):
    ax2.scatter(X_train_std[y_train == l, 0], X_train_std[y_train == l, 1],
                color=c,
                label='class %s' % l,
                alpha=0.5,
                marker=m
                )

ax1.set_title('Training dataset after PCA')
ax2.set_title('Standardized training dataset after PCA')

for ax in (ax1, ax2):
    ax.set_xlabel('1st principal component')
    ax.set_ylabel('2nd principal component')
    ax.legend(loc='upper right')
    ax.grid()

plt.tight_layout()

plt.show()


'''
Created on Feb 16, 2011
k Means Clustering for Ch10 of Machine Learning in Action
@author: Peter Harrington
'''
from numpy import *

def loadDataSet(fileName):      #general function to parse tab -delimited floats
    dataMat = []                #assume last column is target value
    fr = open(fileName)
    for line in fr.readlines():
        curLine = line.strip().split('\t')
        fltLine = map(float,curLine) #map all elements to float()
        dataMat.append(fltLine)
    return dataMat

def distEclud(vecA, vecB):
    return sqrt(sum(power(vecA - vecB, 2))) #la.norm(vecA-vecB)

def randCent(dataSet, k):
    n = shape(dataSet)[1]
    centroids = mat(zeros((k,n)))#create centroid mat
    for j in range(n):#create random cluster centers, within bounds of each dimension
        minJ = min(dataSet[:,j])
        rangeJ = float(max(dataSet[:,j]) - minJ)
        centroids[:,j] = mat(minJ + rangeJ * random.rand(k,1))
    return centroids

def kMeans(dataSet, k, distMeas=distEclud, createCent=randCent):
    m = shape(dataSet)[0]
    clusterAssment = mat(zeros((m,2)))#create mat to assign data points
                                      #to a centroid, also holds SE of each point
    centroids = createCent(dataSet, k)
    clusterChanged = True
    while clusterChanged:
        clusterChanged = False
        for i in range(m):#for each data point assign it to the closest centroid
            minDist = inf; minIndex = -1
            for j in range(k):
                distJI = distMeas(centroids[j,:],dataSet[i,:])
                if distJI < minDist:
                    minDist = distJI; minIndex = j
            if clusterAssment[i,0] != minIndex: clusterChanged = True
            clusterAssment[i,:] = minIndex,minDist**2
        print centroids
        for cent in range(k):#recalculate centroids
            ptsInClust = dataSet[nonzero(clusterAssment[:,0].A==cent)[0]]#get all the point in this cluster
            centroids[cent,:] = mean(ptsInClust, axis=0) #assign centroid to mean
    return centroids, clusterAssment

def biKmeans(dataSet, k, distMeas=distEclud):
    m = shape(dataSet)[0]
    clusterAssment = mat(zeros((m,2)))
    centroid0 = mean(dataSet, axis=0).tolist()[0]
    centList =[centroid0] #create a list with one centroid
    for j in range(m):#calc initial Error
        clusterAssment[j,1] = distMeas(mat(centroid0), dataSet[j,:])**2
    while (len(centList) < k):
        lowestSSE = inf
        for i in range(len(centList)):
            ptsInCurrCluster = dataSet[nonzero(clusterAssment[:,0].A==i)[0],:]#get the data points currently in cluster i
            centroidMat, splitClustAss = kMeans(ptsInCurrCluster, 2, distMeas)
            sseSplit = sum(splitClustAss[:,1])#compare the SSE to the currrent minimum
            sseNotSplit = sum(clusterAssment[nonzero(clusterAssment[:,0].A!=i)[0],1])
            print "sseSplit, and notSplit: ",sseSplit,sseNotSplit
            if (sseSplit + sseNotSplit) < lowestSSE:
                bestCentToSplit = i
                bestNewCents = centroidMat
                bestClustAss = splitClustAss.copy()
                lowestSSE = sseSplit + sseNotSplit
        bestClustAss[nonzero(bestClustAss[:,0].A == 1)[0],0] = len(centList) #change 1 to 3,4, or whatever
        bestClustAss[nonzero(bestClustAss[:,0].A == 0)[0],0] = bestCentToSplit
        print 'the bestCentToSplit is: ',bestCentToSplit
        print 'the len of bestClustAss is: ', len(bestClustAss)
        centList[bestCentToSplit] = bestNewCents[0,:].tolist()[0]#replace a centroid with two best centroids
        centList.append(bestNewCents[1,:].tolist()[0])
        clusterAssment[nonzero(clusterAssment[:,0].A == bestCentToSplit)[0],:]= bestClustAss#reassign new clusters, and SSE
    return mat(centList), clusterAssment

import urllib
import json
def geoGrab(stAddress, city):
    apiStem = 'http://where.yahooapis.com/geocode?'  #create a dict and constants for the goecoder
    params = {}
    params['flags'] = 'J'#JSON return type
    params['appid'] = 'aaa0VN6k'
    params['location'] = '%s %s' % (stAddress, city)
    url_params = urllib.urlencode(params)
    yahooApi = apiStem + url_params      #print url_params
    print yahooApi
    c=urllib.urlopen(yahooApi)
    return json.loads(c.read())

from time import sleep
def massPlaceFind(fileName):
    fw = open('places.txt', 'w')
    for line in open(fileName).readlines():
        line = line.strip()
        lineArr = line.split('\t')
        retDict = geoGrab(lineArr[1], lineArr[2])
        if retDict['ResultSet']['Error'] == 0:
            lat = float(retDict['ResultSet']['Results'][0]['latitude'])
            lng = float(retDict['ResultSet']['Results'][0]['longitude'])
            print "%s\t%f\t%f" % (lineArr[0], lat, lng)
            fw.write('%s\t%f\t%f\n' % (line, lat, lng))
        else: print "error fetching"
        sleep(1)
    fw.close()

def distSLC(vecA, vecB):#Spherical Law of Cosines
    a = sin(vecA[0,1]*pi/180) * sin(vecB[0,1]*pi/180)
    b = cos(vecA[0,1]*pi/180) * cos(vecB[0,1]*pi/180) * \
                      cos(pi * (vecB[0,0]-vecA[0,0]) /180)
    return arccos(a + b)*6371.0 #pi is imported with numpy

import matplotlib
import matplotlib.pyplot as plt
def clusterClubs(numClust=5):
    datList = []
    for line in open('places.txt').readlines():
        lineArr = line.split('\t')
        datList.append([float(lineArr[4]), float(lineArr[3])])
    datMat = mat(datList)
    myCentroids, clustAssing = biKmeans(datMat, numClust, distMeas=distSLC)
    fig = plt.figure()
    rect=[0.1,0.1,0.8,0.8]
    scatterMarkers=['s', 'o', '^', '8', 'p', \
                    'd', 'v', 'h', '>', '<']
    axprops = dict(xticks=[], yticks=[])
    ax0=fig.add_axes(rect, label='ax0', **axprops)
    imgP = plt.imread('Portland.png')
    ax0.imshow(imgP)
    ax1=fig.add_axes(rect, label='ax1', frameon=False)
    for i in range(numClust):
        ptsInCurrCluster = datMat[nonzero(clustAssing[:,0].A==i)[0],:]
        markerStyle = scatterMarkers[i % len(scatterMarkers)]
        ax1.scatter(ptsInCurrCluster[:,0].flatten().A[0], ptsInCurrCluster[:,1].flatten().A[0], marker=markerStyle, s=90)
    ax1.scatter(myCentroids[:,0].flatten().A[0], myCentroids[:,1].flatten().A[0], marker='+', s=300)
    plt.show()


# -*- coding: utf-8 -*-
"""
Created on Sun Oct 16 18:37:23 2011

@author: jev
"""

from urllib import urlretrieve
from urllib2 import urlopen
from pandas import Index, DataFrame
from datetime import datetime
import matplotlib.pyplot as plt

sDate = (2005,1,1)
eDate = (2011,10,1)

symbol = 'SPY'

fName = symbol+'.csv'

try:    # try to load saved csv file, otherwise get from the net
    fid = open(fName)
    lines = fid.readlines()
    fid.close()
    print 'Loaded from ' , fName
except Exception as e:
    print e
    urlStr = 'http://ichart.finance.yahoo.com/table.csv?s={0}&a={1}&b={2}&c={3}&d={4}&e={5}&f={6}'.\
    format(symbol.upper(),sDate[1]-1,sDate[2],sDate[0],eDate[1]-1,eDate[2],eDate[0])
    print 'Downloading from ', urlStr
    urlretrieve(urlStr,symbol+'.csv')
    lines = urlopen(urlStr).readlines()

dates = []
data = [[] for i in range(6)]
#high

# header : Date,Open,High,Low,Close,Volume,Adj Close
for line in lines[1:]:
    fields = line.rstrip().split(',')
    dates.append(datetime.strptime( fields[0],'%Y-%m-%d'))
    for i,field in enumerate(fields[1:]):
        data[i].append(float(field))

idx = Index(dates)
data = dict(zip(['open','high','low','close','volume','adj_close'],data))

# create a pandas dataframe structure
df = DataFrame(data,index=idx).sort()

df.plot(secondary_y=['volume'])


#############################################################################
#                                                                           #
#  This file is part of Verificarlo.                                        #
#                                                                           #
#  Copyright (c) 2015-2021                                                  #
#     Verificarlo contributors                                              #
#     Universite de Versailles St-Quentin-en-Yvelines                       #
#     CMLA, Ecole Normale Superieure de Cachan                              #
#                                                                           #
#  Verificarlo is free software: you can redistribute it and/or modify      #
#  it under the terms of the GNU General Public License as published by     #
#  the Free Software Foundation, either version 3 of the License, or        #
#  (at your option) any later version.                                      #
#                                                                           #
#  Verificarlo is distributed in the hope that it will be useful,           #
#  but WITHOUT ANY WARRANTY; without even the implied warranty of           #
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
#  GNU General Public License for more details.                             #
#                                                                           #
#  You should have received a copy of the GNU General Public License        #
#  along with Verificarlo.  If not, see <http://www.gnu.org/licenses/>.     #
#                                                                           #
#############################################################################

# Manage the view comparing the variables of a run
# Manage the view comparing a variable over different runs
# At its creation, an InspectRuns object will create all the needed Bokeh widgets
# and plots, setup the callback functions (either server side or client side),
# initialize widgets selection, and from this selection generate the first plots.
# Then, when callback functions are triggered, widgets selections are updated,
# and plots are re-generated with the newly selected data.

from math import pi
from functools import partial

import pandas as pd
import numpy as np

from bokeh.plotting import figure, curdoc
from bokeh.embed import components
from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\
    RadioButtonGroup, CheckboxGroup, CustomJS

import helper
import plot

##########################################################################

class InspectRuns:

    # Helper functions related to InspectRun

    def gen_runs_selection(self):
        '''
        Returns a dictionary mapping user-readable strings to all run timestamps
        '''

        runs_dict = {}

        # Iterate over timestamp rows (runs) and fill dict
        for row in self.metadata.iloc:
            # The syntax used by pandas makes this part a bit tricky :
            # row.name is the index of metadata (so it refers to the
            # timestamp), whereas row["name"] is the column called "name"
            # (which is the display string used for the run)

            # runs_dict[run's name] = run's timestamp
            runs_dict[row["name"]] = row.name

        return runs_dict

    def gen_boxplot_tooltips(self, prefix):
        return [
            ("Name", "@%s_x" % prefix),
            ("Min", "@" + prefix + "_min{%0.18e}"),
            ("Max", "@" + prefix + "_max{%0.18e}"),
            ("1st quartile", "@" + prefix + "_quantile25{%0.18e}"),
            ("Median", "@" + prefix + "_quantile50{%0.18e}"),
            ("3rd quartile", "@" + prefix + "_quantile75{%0.18e}"),
            ("μ", "@" + prefix + "_mu{%0.18e}"),
            ("Number of samples (tests)", "@nsamples")
        ]

    def gen_boxplot_tooltips_formatters(self, prefix):
        return {
            "@%s_min" % prefix: "printf",
            "@%s_max" % prefix: "printf",
            "@%s_quantile25" % prefix: "printf",
            "@%s_quantile50" % prefix: "printf",
            "@%s_quantile75" % prefix: "printf",
            "@%s_mu" % prefix: "printf"
        }

    # Data processing helper
    # (computes new distributions for sigma, s2, s10)

    def data_processing(self, dataframe):

        # Compute aggragated mu
        dataframe["mu"] = np.vectorize(
            np.average)(
            dataframe["mu"],
            weights=dataframe["nsamples"])

        # nsamples is the number of aggregated elements (as well as the number
        # of samples for our new sigma and s distributions)
        dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x))

        dataframe["mu_x"] = dataframe.index
        # Make sure that strings don't excede a certain length
        dataframe["mu_x"] = dataframe["mu_x"].apply(
            lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x
        )

        # Get quantiles and mu for sigma, s10, s2
        for prefix in ["sigma", "s10", "s2"]:

            dataframe["%s_x" % prefix] = dataframe["mu_x"]

            dataframe[prefix] = dataframe[prefix].apply(np.sort)

            dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min)
            dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply(
                np.quantile, args=(0.25,))
            dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply(
                np.quantile, args=(0.50,))
            dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply(
                np.quantile, args=(0.75,))
            dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max)
            dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average)
            del dataframe[prefix]

        return dataframe

        # Plots update function

    def update_plots(self):

        groupby_display = self.widgets["groupby_radio"].labels[
            self.widgets["groupby_radio"].active
        ]
        groupby = self.factors_dict[groupby_display]

        filterby_display = self.widgets["filterby_radio"].labels[
            self.widgets["filterby_radio"].active
        ]
        filterby = self.factors_dict[filterby_display]

        # Groupby and aggregate lines belonging to the same group in lists

        groups = self.run_data[
            self.run_data.index.isin(
                [self.widgets["select_filter"].value],
                level=filterby
            )
        ].groupby(groupby)

        groups = groups.agg({
            "sigma": lambda x: x.tolist(),
            "s10": lambda x: x.tolist(),
            "s2": lambda x: x.tolist(),

            "mu": lambda x: x.tolist(),

            # Used for mu weighted average first, then will be replaced
            "nsamples": lambda x: x.tolist()
        })

        # Compute the new distributions, ...
        groups = self.data_processing(groups).to_dict("list")

        # Update source

        # Assign each ColumnDataSource, starting with the boxplots
        for prefix in ["sigma", "s10", "s2"]:

            dict = {
                "%s_x" % prefix: groups["%s_x" % prefix],
                "%s_min" % prefix: groups["%s_min" % prefix],
                "%s_quantile25" % prefix: groups["%s_quantile25" % prefix],
                "%s_quantile50" % prefix: groups["%s_quantile50" % prefix],
                "%s_quantile75" % prefix: groups["%s_quantile75" % prefix],
                "%s_max" % prefix: groups["%s_max" % prefix],
                "%s_mu" % prefix: groups["%s_mu" % prefix],

                "nsamples": groups["nsamples"]
            }

            # Filter outliers if the box is checked
            if len(self.widgets["outliers_filtering_inspect"].active) > 0:

                # Boxplots will be filtered by max then min
                top_outliers = helper.detect_outliers(dict["%s_max" % prefix])
                helper.remove_boxplot_outliers(dict, top_outliers, prefix)

                bottom_outliers = helper.detect_outliers(
                    dict["%s_min" % prefix])
                helper.remove_boxplot_outliers(dict, bottom_outliers, prefix)

            self.sources["%s_source" % prefix].data = dict

        # Finish with the mu plot
        dict = {
            "mu_x": groups["mu_x"],
            "mu": groups["mu"],

            "nsamples": groups["nsamples"]
        }

        self.sources["mu_source"].data = dict

        # Filter outliers if the box is checked
        if len(self.widgets["outliers_filtering_inspect"].active) > 0:
            mu_outliers = helper.detect_outliers(groups["mu"])
            groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers)
            groups["mu_x"] = helper.remove_outliers(
                groups["mu_x"], mu_outliers)

            # Update plots axis/titles

        # Get display string of the last (unselected) factor
        factors_dict = self.factors_dict.copy()
        del factors_dict[groupby_display]
        del factors_dict[filterby_display]
        for_all = list(factors_dict.keys())[0]

        # Update all display strings for plot title (remove caps, plural)
        groupby_display = groupby_display.lower()
        filterby_display = filterby_display.lower()[:-1]
        for_all = for_all.lower()

        self.plots["mu_inspect"].title.text = \
            "Empirical average μ of %s (groupped by %s, for all %s)" \
            % (filterby_display, groupby_display, for_all)

        self.plots["sigma_inspect"].title.text = \
            "Standard deviation σ of %s (groupped by %s, for all %s)" \
            % (filterby_display, groupby_display, for_all)

        self.plots["s10_inspect"].title.text = \
            "Significant digits s of %s (groupped by %s, for all %s)" \
            % (filterby_display, groupby_display, for_all)

        self.plots["s2_inspect"].title.text = \
            "Significant digits s of %s (groupped by %s, for all %s)" \
            % (filterby_display, groupby_display, for_all)

        helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"])
        helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"])
        helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"])
        helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"])

        # Widets' callback functions

    # Run selector callback

    def update_run(self, attrname, old, new):

        filterby = self.widgets["filterby_radio"].labels[
            self.widgets["filterby_radio"].active
        ]
        filterby = self.factors_dict[filterby]

        # Update run selection (by using dict mapping)
        self.current_run = self.runs_dict[new]

        # Update run data
        self.run_data = self.data[self.data["timestamp"] == self.current_run]

        # Save old selected option
        old_value = self.widgets["select_filter"].value

        # Update filter options
        options = self.run_data.index\
            .get_level_values(filterby).drop_duplicates().tolist()
        self.widgets["select_filter"].options = options

        if old_value not in self.widgets["select_filter"].options:
            self.widgets["select_filter"].value = options[0]
            # The update_var callback will be triggered by the assignment

        else:
            # Trigger the callback manually (since the plots need to be updated
            # anyway)
            self.update_filter("", "", old_value)

    # "Group by" radio

    def update_groupby(self, attrname, old, new):

        # Update "Filter by" radio list
        filterby_list = list(self.factors_dict.keys())
        del filterby_list[self.widgets["groupby_radio"].active]
        self.widgets["filterby_radio"].labels = filterby_list

        filterby = self.widgets["filterby_radio"].labels[
            self.widgets["filterby_radio"].active
        ]
        filterby = self.factors_dict[filterby]

        # Save old selected option
        old_value = self.widgets["select_filter"].value

        # Update filter options
        options = self.run_data.index\
            .get_level_values(filterby).drop_duplicates().tolist()
        self.widgets["select_filter"].options = options

        if old_value not in self.widgets["select_filter"].options:
            self.widgets["select_filter"].value = options[0]
            # The update_var callback will be triggered by the assignment

        else:
            # Trigger the callback manually (since the plots need to be updated
            # anyway)
            self.update_filter("", "", old_value)

    # "Filter by" radio

    def update_filterby(self, attrname, old, new):

        filterby = self.widgets["filterby_radio"].labels[
            self.widgets["filterby_radio"].active
        ]
        filterby = self.factors_dict[filterby]

        # Save old selected option
        old_value = self.widgets["select_filter"].value

        # Update filter selector options
        options = self.run_data.index\
            .get_level_values(filterby).drop_duplicates().tolist()
        self.widgets["select_filter"].options = options

        if old_value not in self.widgets["select_filter"].options:
            self.widgets["select_filter"].value = options[0]
            # The update_var callback will be triggered by the assignment

        else:
            # Trigger the callback manually (since the plots need to be updated
            # anyway)
            self.update_filter("", "", old_value)

    # Filter selector callback

    def update_filter(self, attrname, old, new):
        self.update_plots()

    # Filter outliers checkbox callback

    def update_outliers_filtering(self, attrname, old, new):
        # The status (checked/unchecked) of the checkbox is also verified inside
        # self.update_plots(), so calling this function is enough
        self.update_plots()

        # Bokeh setup functions
        # (for both variable and backend selection at once)

    def setup_plots(self):

        tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save"

        # Tooltips and formatters

        dotplot_tooltips = [
            ("Name", "@mu_x"),
            ("μ", "@mu{%0.18e}"),
            ("Number of samples (tests)", "@nsamples")
        ]
        dotplot_formatters = {
            "@mu": "printf"
        }

        sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma")
        sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
            "sigma")

        s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10")
        s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
            "s10")

        s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2")
        s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
            "s2")

        # Plots

        # Mu plot
        self.plots["mu_inspect"] = figure(
            name="mu_inspect",
            title="",
            plot_width=900, plot_height=400, x_range=[""],
            tools=tools, sizing_mode="scale_width"
        )
        plot.fill_dotplot(
            self.plots["mu_inspect"], self.sources["mu_source"], "mu",
            tooltips=dotplot_tooltips,
            tooltips_formatters=dotplot_formatters
        )
        self.doc.add_root(self.plots["mu_inspect"])

        # Sigma plot
        self.plots["sigma_inspect"] = figure(
            name="sigma_inspect",
            title="",
            plot_width=900, plot_height=400, x_range=[""],
            tools=tools, sizing_mode="scale_width"
        )
        plot.fill_boxplot(
            self.plots["sigma_inspect"],
            self.sources["sigma_source"],
            prefix="sigma",
            tooltips=sigma_boxplot_tooltips,
            tooltips_formatters=sigma_boxplot_tooltips_formatters)
        self.doc.add_root(self.plots["sigma_inspect"])

        # s plots
        self.plots["s10_inspect"] = figure(
            name="s10_inspect",
            title="",
            plot_width=900, plot_height=400, x_range=[""],
            tools=tools, sizing_mode='scale_width'
        )
        plot.fill_boxplot(
            self.plots["s10_inspect"],
            self.sources["s10_source"],
            prefix="s10",
            tooltips=s10_boxplot_tooltips,
            tooltips_formatters=s10_boxplot_tooltips_formatters)
        s10_tab_inspect = Panel(
            child=self.plots["s10_inspect"],
            title="Base 10")

        self.plots["s2_inspect"] = figure(
            name="s2_inspect",
            title="",
            plot_width=900, plot_height=400, x_range=[""],
            tools=tools, sizing_mode='scale_width'
        )
        plot.fill_boxplot(
            self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2",
            tooltips=s2_boxplot_tooltips,
            tooltips_formatters=s2_boxplot_tooltips_formatters
        )
        s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2")

        s_tabs_inspect = Tabs(
            name="s_tabs_inspect",
            tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location="below"
        )
        self.doc.add_root(s_tabs_inspect)

    def setup_widgets(self):

        # Generation of selectable items

        # Dict contains all inspectable runs (maps display strings to timestamps)
        # The dict structure allows to get the timestamp from the display string
        # in O(1)
        self.runs_dict = self.gen_runs_selection()

        # Dict maps display strings to column names for the different factors
        # (var, backend, test)
        self.factors_dict = {
            "Variables": "variable",
            "Backends": "vfc_backend",
            "Tests": "test"
        }

        # Run selection

        # Contains all options strings
        runs_display = list(self.runs_dict.keys())
        # Will be used when updating plots (contains actual number)
        self.current_run = self.runs_dict[runs_display[-1]]
        # Contains the selected option string, used to update current_n_runs
        current_run_display = runs_display[-1]
        # This contains only entries matching the run
        self.run_data = self.data[self.data["timestamp"] == self.current_run]

        change_run_callback_js = "updateRunMetadata(cb_obj.value);"

        self.widgets["select_run"] = Select(
            name="select_run", title="Run :",
            value=current_run_display, options=runs_display
        )
        self.doc.add_root(self.widgets["select_run"])
        self.widgets["select_run"].on_change("value", self.update_run)
        self.widgets["select_run"].js_on_change("value", CustomJS(
            code=change_run_callback_js,
            args=(dict(
                metadata=helper.metadata_to_dict(
                    helper.get_metadata(self.metadata, self.current_run)
                )
            ))
        ))

        # Factors selection

        # "Group by" radio
        self.widgets["groupby_radio"] = RadioButtonGroup(
            name="groupby_radio",
            labels=list(self.factors_dict.keys()), active=0
        )
        self.doc.add_root(self.widgets["groupby_radio"])
        # The functions are defined inside the template to avoid writing too
        # much JS server side
        self.widgets["groupby_radio"].on_change(
            "active",
            self.update_groupby
        )

        # "Filter by" radio
        # Get all possible factors, and remove the one selected in "Group by"
        filterby_list = list(self.factors_dict.keys())
        del filterby_list[self.widgets["groupby_radio"].active]

        self.widgets["filterby_radio"] = RadioButtonGroup(
            name="filterby_radio",
            labels=filterby_list, active=0
        )
        self.doc.add_root(self.widgets["filterby_radio"])
        # The functions are defined inside the template to avoid writing too
        # much JS server side
        self.widgets["filterby_radio"].on_change(
            "active",
            self.update_filterby
        )

        # Filter selector

        filterby = self.widgets["filterby_radio"].labels[
            self.widgets["filterby_radio"].active
        ]
        filterby = self.factors_dict[filterby]

        options = self.run_data.index\
            .get_level_values(filterby).drop_duplicates().tolist()

        self.widgets["select_filter"] = Select(
            # We need a different name to avoid collision in the template with
            # the runs comparison's widget
            name="select_filter", title="Select a filter :",
            value=options[0], options=options
        )
        self.doc.add_root(self.widgets["select_filter"])
        self.widgets["select_filter"]\
            .on_change("value", self.update_filter)

        # Toggle for outliers filtering

        self.widgets["outliers_filtering_inspect"] = CheckboxGroup(
            name="outliers_filtering_inspect",
            labels=["Filter outliers"], active=[]
        )
        self.doc.add_root(self.widgets["outliers_filtering_inspect"])
        self.widgets["outliers_filtering_inspect"]\
            .on_change("active", self.update_outliers_filtering)

        # Communication methods
        # (to send/receive messages to/from master)

    def change_repo(self, new_data, new_metadata):
        '''
        When received, update data and metadata with the new repo, and update
        everything
        '''

        self.data = new_data
        self.metadata = new_metadata

        self.runs_dict = self.gen_runs_selection()

        runs_display = list(self.runs_dict.keys())
        current_run_display = runs_display[-1]

        # Update widget (and trigger its callback)
        self.widgets["select_run"].options = runs_display
        self.widgets["select_run"].value = current_run_display

        filterby = self.widgets["filterby_radio"].labels[
            self.widgets["filterby_radio"].active
        ]
        filterby = self.factors_dict[filterby]

        self.run_data = self.data[self.data["timestamp"] == self.current_run]
        options = self.run_data.index\
            .get_level_values(filterby).drop_duplicates().tolist()

        # Update widget (and trigger its callback)
        self.widgets["select_filter"].options = options
        self.widgets["select_filter"].value = options[0]

    def switch_view(self, run_name):
        '''When received, switch selected run to run_name'''

        # This will trigger the widget's callback
        self.widgets["select_run"].value = run_name

        # Constructor

    def __init__(self, master, doc, data, metadata):
        '''
        Here are the most important attributes of the InspectRuns class

        master : reference to the ViewMaster class
        doc : an object provided by Bokeh to add elements to the HTML document
        data : pandas dataframe containing all the tests data
        metadata : pandas dataframe containing all the tests metadata

        sources : ColumnDataSource object provided by Bokeh, contains current
        data for the plots (inside the .data attribute)
        plots : dictionary of Bokeh plots
        widgets : dictionary of Bokeh widgets
        '''

        self.master = master

        self.doc = doc
        self.data = data
        self.metadata = metadata

        self.sources = {
            "mu_source": ColumnDataSource(data={}),
            "sigma_source": ColumnDataSource(data={}),
            "s10_source": ColumnDataSource(data={}),
            "s2_source": ColumnDataSource(data={})
        }

        self.plots = {}
        self.widgets = {}

        # Setup Bokeh objects
        self.setup_plots()
        self.setup_widgets()

        # Pass the initial metadata to the template (will be updated in CustomJS
        # callbacks). This is required because metadata is not displayed in a
        # Bokeh widget, so we can't update this with a server callback.
        initial_run = helper.get_metadata(self.metadata, self.current_run)
        self.doc.template_variables["initial_timestamp"] = initial_run.name
        self.doc.template_variables["initial_repo"] = initial_run.repo_name

        # At this point, everything should have been initialized, so we can
        # show the plots for the first time
        self.update_plots()


import os
import shutil
import tempfile
import warnings
import nose
import numpy

from sklearn.datasets import get_data_home
from sklearn.datasets import clear_data_home
from sklearn.datasets import load_files
from sklearn.datasets import load_sample_images
from sklearn.datasets import load_sample_image
from sklearn.datasets import load_digits
from sklearn.datasets import load_diabetes
from sklearn.datasets import load_linnerud
from sklearn.datasets import load_iris
from sklearn.datasets import load_boston

from sklearn.externals.six import b, u

from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises

DATA_HOME = tempfile.mkdtemp(prefix="scikit_learn_data_home_test_")
LOAD_FILES_ROOT = tempfile.mkdtemp(prefix="scikit_learn_load_files_test_")
TEST_CATEGORY_DIR1 = ""
TEST_CATEGORY_DIR2 = ""

def _remove_dir(path):
    if os.path.isdir(path):
        shutil.rmtree(path)

def teardown_module():
    """Test fixture (clean up) run once after all tests of this module"""
    for path in [DATA_HOME, LOAD_FILES_ROOT]:
        _remove_dir(path)

def setup_load_files():
    global TEST_CATEGORY_DIR1
    global TEST_CATEGORY_DIR2
    TEST_CATEGORY_DIR1 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    TEST_CATEGORY_DIR2 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    sample_file = tempfile.NamedTemporaryFile(dir=TEST_CATEGORY_DIR1,
                                              delete=False)
    sample_file.write(b("Hello World!\n"))
    sample_file.close()

def teardown_load_files():
    _remove_dir(TEST_CATEGORY_DIR1)
    _remove_dir(TEST_CATEGORY_DIR2)

def test_data_home():
    # get_data_home will point to a pre-existing folder
    data_home = get_data_home(data_home=DATA_HOME)
    assert_equal(data_home, DATA_HOME)
    assert_true(os.path.exists(data_home))

    # clear_data_home will delete both the content and the folder it-self
    clear_data_home(data_home=data_home)
    assert_false(os.path.exists(data_home))

    # if the folder is missing it will be created again
    data_home = get_data_home(data_home=DATA_HOME)
    assert_true(os.path.exists(data_home))

def test_default_empty_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 0)
    assert_equal(len(res.target_names), 0)
    assert_equal(res.DESCR, None)

@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_default_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.data, [b("Hello World!\n")])

@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_load_files_w_categories_desc_and_encoding():
    category = os.path.abspath(TEST_CATEGORY_DIR1).split('/').pop()
    res = load_files(LOAD_FILES_ROOT, description="test",
                     categories=category, encoding="utf-8")
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 1)
    assert_equal(res.DESCR, "test")
    assert_equal(res.data, [u("Hello World!\n")])

@nose.tools.with_setup(setup_load_files, teardown_load_files)
def test_load_files_wo_load_content():
    res = load_files(LOAD_FILES_ROOT, load_content=False)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.get('data'), None)

def test_load_sample_images():
    try:
        res = load_sample_images()
        assert_equal(len(res.images), 2)
        assert_equal(len(res.filenames), 2)
        assert_true(res.DESCR)
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_digits():
    digits = load_digits()
    assert_equal(digits.data.shape, (1797, 64))
    assert_equal(numpy.unique(digits.target).size, 10)

def test_load_digits_n_class_lt_10():
    digits = load_digits(9)
    assert_equal(digits.data.shape, (1617, 64))
    assert_equal(numpy.unique(digits.target).size, 9)

def test_load_sample_image():
    try:
        china = load_sample_image('china.jpg')
        assert_equal(china.dtype, 'uint8')
        assert_equal(china.shape, (427, 640, 3))
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_missing_sample_image_error():
    have_PIL = True
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
    except ImportError:
        have_PIL = False
    if have_PIL:
        assert_raises(AttributeError, load_sample_image,
                      'blop.jpg')
    else:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_diabetes():
    res = load_diabetes()
    assert_equal(res.data.shape, (442, 10))
    assert_true(res.target.size, 442)

def test_load_linnerud():
    res = load_linnerud()
    assert_equal(res.data.shape, (20, 3))
    assert_equal(res.target.shape, (20, 3))
    assert_equal(len(res.target_names), 3)
    assert_true(res.DESCR)

def test_load_iris():
    res = load_iris()
    assert_equal(res.data.shape, (150, 4))
    assert_equal(res.target.size, 150)
    assert_equal(res.target_names.size, 3)
    assert_true(res.DESCR)

def test_load_boston():
    res = load_boston()
    assert_equal(res.data.shape, (506, 13))
    assert_equal(res.target.size, 506)
    assert_equal(res.feature_names.size, 13)
    assert_true(res.DESCR)


import sys
import re

import numpy as np
from scipy.sparse import csc_matrix, csr_matrix, lil_matrix
from sklearn.utils.testing import (assert_almost_equal, assert_array_equal,
                                   assert_true)

from sklearn.datasets import load_digits
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.neural_network import BernoulliRBM
from sklearn.utils.validation import assert_all_finite
np.seterr(all='warn')

Xdigits = load_digits().data
Xdigits -= Xdigits.min()
Xdigits /= Xdigits.max()

def test_fit():
    X = Xdigits.copy()

    rbm = BernoulliRBM(n_components=64, learning_rate=0.1,
                       batch_size=10, n_iter=7, random_state=9)
    rbm.fit(X)

    assert_almost_equal(rbm.score_samples(X).mean(), -21., decimal=0)

    # in-place tricks shouldn't have modified X
    assert_array_equal(X, Xdigits)

def test_partial_fit():
    X = Xdigits.copy()
    rbm = BernoulliRBM(n_components=64, learning_rate=0.1,
                       batch_size=20, random_state=9)
    n_samples = X.shape[0]
    n_batches = int(np.ceil(float(n_samples) / rbm.batch_size))
    batch_slices = np.array_split(X, n_batches)

    for i in range(7):
        for batch in batch_slices:
            rbm.partial_fit(batch)

    assert_almost_equal(rbm.score_samples(X).mean(), -21., decimal=0)
    assert_array_equal(X, Xdigits)

def test_transform():
    X = Xdigits[:100]
    rbm1 = BernoulliRBM(n_components=16, batch_size=5,
                        n_iter=5, random_state=42)
    rbm1.fit(X)

    Xt1 = rbm1.transform(X)
    Xt2 = rbm1._mean_hiddens(X)

    assert_array_equal(Xt1, Xt2)

def test_small_sparse():
    # BernoulliRBM should work on small sparse matrices.
    X = csr_matrix(Xdigits[:4])
    BernoulliRBM().fit(X)       # no exception

def test_small_sparse_partial_fit():
    for sparse in [csc_matrix, csr_matrix]:
        X_sparse = sparse(Xdigits[:100])
        X = Xdigits[:100].copy()

        rbm1 = BernoulliRBM(n_components=64, learning_rate=0.1,
                            batch_size=10, random_state=9)
        rbm2 = BernoulliRBM(n_components=64, learning_rate=0.1,
                            batch_size=10, random_state=9)

        rbm1.partial_fit(X_sparse)
        rbm2.partial_fit(X)

        assert_almost_equal(rbm1.score_samples(X).mean(),
                            rbm2.score_samples(X).mean(),
                            decimal=0)

def test_sample_hiddens():
    rng = np.random.RandomState(0)
    X = Xdigits[:100]
    rbm1 = BernoulliRBM(n_components=2, batch_size=5,
                        n_iter=5, random_state=42)
    rbm1.fit(X)

    h = rbm1._mean_hiddens(X[0])
    hs = np.mean([rbm1._sample_hiddens(X[0], rng) for i in range(100)], 0)

    assert_almost_equal(h, hs, decimal=1)

def test_fit_gibbs():
    # Gibbs on the RBM hidden layer should be able to recreate [[0], [1]]
    # from the same input
    rng = np.random.RandomState(42)
    X = np.array([[0.], [1.]])
    rbm1 = BernoulliRBM(n_components=2, batch_size=2,
                        n_iter=42, random_state=rng)
    # you need that much iters
    rbm1.fit(X)
    assert_almost_equal(rbm1.components_,
                        np.array([[0.02649814], [0.02009084]]), decimal=4)
    assert_almost_equal(rbm1.gibbs(X), X)
    return rbm1

def test_fit_gibbs_sparse():
    # Gibbs on the RBM hidden layer should be able to recreate [[0], [1]] from
    # the same input even when the input is sparse, and test against non-sparse
    rbm1 = test_fit_gibbs()
    rng = np.random.RandomState(42)
    from scipy.sparse import csc_matrix
    X = csc_matrix([[0.], [1.]])
    rbm2 = BernoulliRBM(n_components=2, batch_size=2,
                        n_iter=42, random_state=rng)
    rbm2.fit(X)
    assert_almost_equal(rbm2.components_,
                        np.array([[0.02649814], [0.02009084]]), decimal=4)
    assert_almost_equal(rbm2.gibbs(X), X.toarray())
    assert_almost_equal(rbm1.components_, rbm2.components_)

def test_gibbs_smoke():
    # Check if we don't get NaNs sampling the full digits dataset.
    # Also check that sampling again will yield different results.
    X = Xdigits
    rbm1 = BernoulliRBM(n_components=42, batch_size=40,
                        n_iter=20, random_state=42)
    rbm1.fit(X)
    X_sampled = rbm1.gibbs(X)
    assert_all_finite(X_sampled)
    X_sampled2 = rbm1.gibbs(X)
    assert_true(np.all((X_sampled != X_sampled2).max(axis=1)))

def test_score_samples():
    # Test score_samples (pseudo-likelihood) method.
    # Assert that pseudo-likelihood is computed without clipping.
    # See Fabian's blog, http://bit.ly/1iYefRk
    rng = np.random.RandomState(42)
    X = np.vstack([np.zeros(1000), np.ones(1000)])
    rbm1 = BernoulliRBM(n_components=10, batch_size=2,
                        n_iter=10, random_state=rng)
    rbm1.fit(X)
    assert_true((rbm1.score_samples(X) < -300).all())

    # Sparse vs. dense should not affect the output. Also test sparse input
    # validation.
    rbm1.random_state = 42
    d_score = rbm1.score_samples(X)
    rbm1.random_state = 42
    s_score = rbm1.score_samples(lil_matrix(X))
    assert_almost_equal(d_score, s_score)

    # Test numerical stability (#2785): would previously generate infinities
    # and crash with an exception.
    with np.errstate(under='ignore'):
        rbm1.score_samples([np.arange(1000) * 100])

def test_rbm_verbose():
    rbm = BernoulliRBM(n_iter=2, verbose=10)
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        rbm.fit(Xdigits)
    finally:
        sys.stdout = old_stdout

def test_sparse_and_verbose():
    # Make sure RBM works with sparse input when verbose=True
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    from scipy.sparse import csc_matrix
    X = csc_matrix([[0.], [1.]])
    rbm = BernoulliRBM(n_components=2, batch_size=2, n_iter=1,
                       random_state=42, verbose=True)
    try:
        rbm.fit(X)
        s = sys.stdout.getvalue()
        # make sure output is sound
        assert_true(re.match(r"\[BernoulliRBM\] Iteration 1,"
                             r" pseudo-likelihood = -?(\d)+(\.\d+)?,"
                             r" time = (\d|\.)+s",
                             s))
    finally:
        sys.stdout = old_stdout


"""
==========================
Plotting Validation Curves
==========================

In this plot you can see the training scores and validation scores of an SVM
for different values of the kernel parameter gamma. For very low values of
gamma, you can see that both the training score and the validation score are
low. This is called underfitting. Medium values of gamma will result in high
values for both scores, i.e. the classifier is performing fairly well. If gamma
is too high, the classifier will overfit, which means that the training score
is good but the validation score is poor.
"""
print(__doc__)

import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import load_digits
from sklearn.svm import SVC
from sklearn.learning_curve import validation_curve

digits = load_digits()
X, y = digits.data, digits.target

param_range = np.logspace(-6, -1, 5)
train_scores, test_scores = validation_curve(
    SVC(), X, y, param_name="gamma", param_range=param_range,
    cv=10, scoring="accuracy", n_jobs=1)
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)

plt.title("Validation Curve with SVM")
plt.xlabel("$\gamma$")
plt.ylabel("Score")
plt.ylim(0.0, 1.1)
plt.semilogx(param_range, train_scores_mean, label="Training score", color="r")
plt.fill_between(param_range, train_scores_mean - train_scores_std,
                 train_scores_mean + train_scores_std, alpha=0.2, color="r")
plt.semilogx(param_range, test_scores_mean, label="Cross-validation score",
             color="g")
plt.fill_between(param_range, test_scores_mean - test_scores_std,
                 test_scores_mean + test_scores_std, alpha=0.2, color="g")
plt.legend(loc="best")
plt.show()


#!/usr/bin/env python
import click
import os
import codecs
import json
import pandas as pd

from nlppln.utils import create_dirs, get_files

@click.command()
@click.argument('in_dir', type=click.Path(exists=True))
@click.option('--out_dir', '-o', default=os.getcwd(), type=click.Path())
@click.option('--name', '-n', default='ner_stats.csv')
def nerstats(in_dir, out_dir, name):
    create_dirs(out_dir)

    frames = []

    in_files = get_files(in_dir)

    for fi in in_files:
        with codecs.open(fi, encoding='utf-8') as f:
            saf = json.load(f)
        data = {}
        data['word'] = [t['word'] for t in saf['tokens'] if 'ne' in t.keys()]
        data['ner'] = [t['ne'] for t in saf['tokens'] if 'ne' in t.keys()]
        data['w_id'] = [t['id'] for t in saf['tokens'] if 'ne' in t.keys()]
        data['text'] = [os.path.basename(fi)
                        for t in saf['tokens'] if 'ne' in t.keys()]

        frames.append(pd.DataFrame(data=data))

    df = pd.concat(frames, ignore_index=True)
    df.to_csv(os.path.join(out_dir, name), encoding='utf-8')

if __name__ == '__main__':
    nerstats()


import twsi_eval
import argparse
from pandas import read_csv
from twsi_eval import TWSI_INVENTORY, map_sense_inventories, calculate_evaluation_scores

TWSI_DATASET = 'data/Dataset-TWSI-2.csv'

def evaluate_uppper_bound(twsi_dataset_fath, user2twsi):
    print 'Estimating upper bound performance: ', twsi_dataset_fath
    correct = 0
    checked = set()
    predictions = read_csv(twsi_dataset_fath, sep='\t', encoding='utf8')
    i = -1

    for i, row in predictions.iterrows():
        context_id = row.context_id
        gold_sense_ids = unicode(row.gold_sense_ids)
        key = unicode(context_id) + row.target

        if key not in checked:
            checked.add(key)
            if gold_sense_ids in user2twsi[row.target].values():
                correct += 1

    return correct, i+1

def main():
    parser = argparse.ArgumentParser(description='Estimation of the upper bound performance given the custom Word Sense Inventory.')
    parser.add_argument('user_inventory', metavar='sense-inventory', help='word sense inventory file, format:\n word_senseID <tab> list,of,words')
    parser.add_argument('-predictions', help='word sense disambiguation predictions in the 9 column lexical sample format. By default use full Dataset-TWSI-2 set.', default=TWSI_DATASET)
    args = parser.parse_args()

    user2twsi = map_sense_inventories(TWSI_INVENTORY, args.user_inventory)
    correct, count = evaluate_uppper_bound(args.predictions, user2twsi)

    print "\nUpper Bound Results:"
    print "Correct, retrieved, nr_sentences"
    print correct, "\t", count
    precision, recall, fscore = calculate_evaluation_scores(correct=correct, retrieved=correct, itemcount=count)
    print "Precision:", precision, "\tRecall:", recall, "\tF1:", fscore

if __name__ == '__main__':
    main()


"""
Replace corrupted data files with daily data re-downloaded with wget
"""

import sys
sys.path.append('/home/jwalker/dynamics/python/atmos-tools')
sys.path.append('/home/jwalker/dynamics/python/atmos-read')

import os
import shutil
import xarray as xray
import numpy as np
import collections
import time
import matplotlib.pyplot as plt
import pandas as pd
import atmos as atm
import precipdat
import merra

# ----------------------------------------------------------------------
datadir = '/net/eady/data1/jwalker/datastore/merra2/wget/'
savedir = '/net/eady/data1/jwalker/datastore/merra2/merged/'
probdata = pd.read_csv('scripts/merra_urls/merge_data.csv', index_col=0)

# For each corrupted data file:
# - load the corrupted data file
# - load the new downloaded file for the problem day
# - calculate d/dp and other stuff
# - merge the data for the affected day
# - save into data file for the year

def latlon_filestr(lat1, lat2, lon1, lon2):
    """Return nicely formatted string for lat-lon range."""
    latstr = atm.latlon_str(lat1, lat2, 'lat')
    lonstr = atm.latlon_str(lon1, lon2, 'lon')
    return lonstr + '_' + latstr

def latlon_data(var, lat1, lat2, lon1, lon2, plev=None):
    """Extract lat-lon subset of data."""
    name = var.name
    varnm = name
    subset_dict = {'lat' : (lat1, lat2), 'lon' : (lon1, lon2)}
    latlonstr = latlon_filestr(lat1, lat2, lon1, lon2)
    if plev is not None:
        name = name + '%d' % plev
        subset_dict['plev'] = (plev, plev)
    var = atm.subset(var, subset_dict, copy=False, squeeze=True)
    var.name = name
    var.attrs['filestr'] = '%s_%s' % (name, latlonstr)
    var.attrs['varnm'] = varnm
    return var

def pgradient(var, lat1, lat2, lon1, lon2, plev):
    """Return d/dp of a lat-lon variable."""
    pwidth = 100
    p1, p2 = plev - pwidth, plev + pwidth
    var = atm.subset(var, {'lat' : (lat1, lat2), 'lon' : (lon1, lon2),
                           'plev' : (p1, p2)}, copy=False, squeeze=True)
    latlonstr = latlon_filestr(lat1, lat2, lon1, lon2)
    attrs = var.attrs
    pname = atm.get_coord(var, 'plev', 'name')
    pdim = atm.get_coord(var, 'plev', 'dim')
    pres = var[pname]
    pres = atm.pres_convert(pres, pres.attrs['units'], 'Pa')
    dvar_dp = atm.gradient(var, pres, axis=pdim)
    dvar_dp = atm.subset(dvar_dp, {pname : (plev, plev)}, copy=False,
                         squeeze=True)
    varnm = 'D%sDP' % var.name
    name = '%s%d' % (varnm, plev)
    dvar_dp.name = name
    attrs['long_name'] = 'd/dp of ' + var.attrs['long_name']
    attrs['standard_name'] = 'd/dp of ' + var.attrs['standard_name']
    attrs['units'] = ('(%s)/Pa' % attrs['units'])
    attrs[pname] = plev
    attrs['filestr'] = '%s_%s' % (name, latlonstr)
    attrs['varnm'] = varnm
    dvar_dp.attrs = attrs
    return dvar_dp

def var_calcs(filenm, varnm, plev, latlon=(-90, 90, 40, 120)):
    """Process a single variable from a single day."""
    lat1, lat2, lon1, lon2 = latlon
    if varnm == 'DUDP':
        nm, dp = 'U', True
    elif varnm == 'DOMEGADP':
        nm, dp = 'OMEGA', True
    else:
        nm, dp = varnm, False
    with xray.open_dataset(filenm) as ds:
        var = ds[nm].load()
    if dp:
        print('Computing d/dp')
        var = pgradient(var, lat1, lat2, lon1, lon2, plev)
    else:
        var = latlon_data(var, lat1, lat2, lon1, lon2, plev)

    return var

def process_row(row, datadir, savedir):
    filenm1 = row['filename']
    year = row['year']
    varnm = row['varnm']
    plev = row['plev']
    jday = row['jday']
    filenm2 = datadir + row['datfile']
    savefile1 = filenm1
    savefile2 = savedir + os.path.split(filenm1)[1]

    print('%d, %s, plev=%d' % (year, varnm, plev))
    print('Reading original data from ' + filenm1)
    with xray.open_dataset(filenm1) as ds:
        var1 = ds[varnm].load()
    print('Processing new data from ' + filenm2)
    var2 = var_calcs(filenm2, varnm, plev)

    print('Merging data for jday %d' % jday)
    var = var1.copy()
    ind = jday - 1
    days = atm.get_coord(var1, 'day')
    if not days[ind] == jday:
        raise ValueError('Days not indexed from 1, need to edit code to handle')
    var[ind] = var2
    print('Saving to ' + savefile1)
    var.to_netcdf(savefile1)
    print('Saving to ' + savefile2)
    var.to_netcdf(savefile2)

    data = {'orig' : var1, 'new' : var2, 'merged' : var}
    return data

# Make a copy of each of the original files -- only run this code once!
# for filenm in probdata['filename']:
#     shutil.copyfile(filenm, filenm.replace('.nc', '_orig.nc'))

for i, row in probdata.iterrows():
    data = process_row(row, datadir, savedir)

# Plot data to check
def plot_data(probdata, savedir, i):
    row = probdata.iloc[i]
    filenm = row['filename']
    filenm = savedir + os.path.split(filenm)[1]
    jday = row['jday']
    varnm = row['varnm']
    with xray.open_dataset(filenm) as ds:
        var = ds[varnm].load()
    plt.figure(figsize=(16, 8))
    plt.suptitle(os.path.split(filenm)[1])
    plt.subplot(1, 3, 1)
    atm.pcolor_latlon(var.sel(day=(jday-1)))
    plt.title(jday - 1)
    plt.subplot(1, 3, 2)
    atm.pcolor_latlon(var.sel(day=jday))
    plt.title(jday)
    plt.subplot(1, 3, 3)
    atm.pcolor_latlon(var.sel(day=(jday+1)))
    plt.title(jday + 1)


from pylab import *
import astronomy as ast

# to format the labels better
from matplotlib.ticker import FormatStrFormatter
fmt = FormatStrFormatter('%1.2g')  # or whatever

X1 = load('ec2117ans_1_c.dat')
x1 = X1[:,0]
y1 = 10**(X1[:,2]/(-2.5))
y1 /= average(y1)

T0 = 2453964.3307097
P = 0.1545255

figure(figsize=(6,4))
subplots_adjust(hspace=0.6,left=0.16)

ax = subplot(211)

#plot(x1,y1,'.')
scatter((x1-T0)/P,y1,s=0.8,faceted=False)
xlabel('Orbital Phase')
ylabel('Intensity')
title('Original Lightcurve')
#ylim(min(y1)-0.0000005,max(y1)+0.0000005)
ax.yaxis.set_major_formatter(fmt)

ax = subplot(212)

x2,y2 = ast.signal.dft(x1,y1,0,7000,1)

plot(x2,y2,'k-')
xlabel('Frequency (cycles/day)')
ylabel('Amplitude')
#vlines(3560,0.000000025,0.00000003,color='k',linestyle='solid')
#vlines(950,0.000000025,0.00000003,color='k',linestyle='solid')
#text(3350,0.000000035,'DNO',fontsize=10)
#text(700,0.000000035,'lpDNO',fontsize=10)
xlim(0,7000)
ylim(0,0.004)
title('Periodogram')

#ax.yaxis.set_major_formatter(fmt)

savefig('unflattened.png')

show()


import time

import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np

from pySDC.implementations.datatype_classes.mesh import mesh, imex_mesh
from pySDC.implementations.problem_classes.AllenCahn_2D_FD import allencahn_fullyimplicit, allencahn_semiimplicit

# http://www.personal.psu.edu/qud2/Res/Pre/dz09sisc.pdf

def setup_problem():

    problem_params = dict()
    problem_params['nu'] = 2
    problem_params['nvars'] = (128, 128)
    problem_params['eps'] = 0.04
    problem_params['newton_maxiter'] = 100
    problem_params['newton_tol'] = 1E-07
    problem_params['lin_tol'] = 1E-08
    problem_params['lin_maxiter'] = 100
    problem_params['radius'] = 0.25

    return problem_params

def run_implicit_Euler(t0, dt, Tend):
    """
    Routine to run particular SDC variant

    Args:
        Tend (float): end time for dumping
    """

    problem = allencahn_fullyimplicit(problem_params=setup_problem(), dtype_u=mesh, dtype_f=mesh)

    u = problem.u_exact(t0)

    radius = []
    exact_radius = []
    nsteps = int((Tend - t0) / dt)
    startt = time.time()
    t = t0
    for n in range(nsteps):

        u_new = problem.solve_system(rhs=u, factor=dt, u0=u, t=t)

        u = u_new
        t += dt

        r, re = compute_radius(u, problem.dx, t, problem.params.radius)
        radius.append(r)
        exact_radius.append(re)

        print(' ... done with time = %6.4f, step = %i / %i' % (t, n + 1, nsteps))

    print('Time to solution: %6.4f sec.' % (time.time() - startt))

    fname = 'data/AC_reference_Tend{:.1e}'.format(Tend) + '.npz'
    loaded = np.load(fname)
    uref = loaded['uend']

    err = np.linalg.norm(uref - u, np.inf)
    print('Error vs. reference solution: %6.4e' % err)

    return err, radius, exact_radius

def run_imex_Euler(t0, dt, Tend):
    """
    Routine to run particular SDC variant

    Args:
        Tend (float): end time for dumping
    """

    problem = allencahn_semiimplicit(problem_params=setup_problem(), dtype_u=mesh, dtype_f=imex_mesh)

    u = problem.u_exact(t0)

    radius = []
    exact_radius = []
    nsteps = int((Tend - t0) / dt)
    startt = time.time()
    t = t0
    for n in range(nsteps):

        f = problem.eval_f(u, t)
        rhs = u + dt * f.expl
        u_new = problem.solve_system(rhs=rhs, factor=dt, u0=u, t=t)

        u = u_new
        t += dt

        r, re = compute_radius(u, problem.dx, t, problem.params.radius)
        radius.append(r)
        exact_radius.append(re)

        print(' ... done with time = %6.4f, step = %i / %i' % (t, n + 1, nsteps))

    print('Time to solution: %6.4f sec.' % (time.time() - startt))

    fname = 'data/AC_reference_Tend{:.1e}'.format(Tend) + '.npz'
    loaded = np.load(fname)
    uref = loaded['uend']

    err = np.linalg.norm(uref - u, np.inf)
    print('Error vs. reference solution: %6.4e' % err)

    return err, radius, exact_radius

def run_CrankNicholson(t0, dt, Tend):
    """
    Routine to run particular SDC variant

    Args:
        Tend (float): end time for dumping
    """

    problem = allencahn_fullyimplicit(problem_params=setup_problem(), dtype_u=mesh, dtype_f=mesh)

    u = problem.u_exact(t0)

    radius = []
    exact_radius = []
    nsteps = int((Tend - t0)/dt)
    startt = time.time()
    t = t0
    for n in range(nsteps):

        rhs = u + dt / 2 * problem.eval_f(u, t)
        u_new = problem.solve_system(rhs=rhs, factor=dt / 2, u0=u, t=t)

        u = u_new
        t += dt

        r, re = compute_radius(u, problem.dx, t, problem.params.radius)
        radius.append(r)
        exact_radius.append(re)

        print(' ... done with time = %6.4f, step = %i / %i' % (t, n + 1, nsteps))

    print('Time to solution: %6.4f sec.' % (time.time() - startt))

    fname = 'data/AC_reference_Tend{:.1e}'.format(Tend) + '.npz'
    loaded = np.load(fname)
    uref = loaded['uend']

    err = np.linalg.norm(uref - u, np.inf)
    print('Error vs. reference solution: %6.4e' % err)

    return err, radius, exact_radius

def compute_radius(u, dx, t, init_radius):

    c = np.count_nonzero(u >= 0.0)
    radius = np.sqrt(c / np.pi) * dx

    exact_radius = np.sqrt(max(init_radius ** 2 - 2.0 * t, 0))

    return radius, exact_radius

def plot_radius(xcoords, exact_radius, radii):

    fig, ax = plt.subplots()
    plt.plot(xcoords, exact_radius, color='k', linestyle='--', linewidth=1, label='exact')

    for type, radius in radii.items():
        plt.plot(xcoords, radius, linestyle='-', linewidth=2, label=type)

    ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%1.2f'))
    ax.set_ylabel('radius')
    ax.set_xlabel('time')
    ax.grid()
    ax.legend(loc=3)
    fname = 'data/AC_contracting_circle_standard_integrators'
    plt.savefig('{}.pdf'.format(fname),  bbox_inches='tight')

    # plt.show()

def main_radius(cwd=''):
    """
    Main driver

    Args:
        cwd (str): current working directory (need this for testing)
    """

    # setup parameters "in time"
    t0 = 0.0
    dt = 0.001
    Tend = 0.032

    radii = {}
    _, radius, exact_radius = run_implicit_Euler(t0=t0, dt=dt, Tend=Tend)
    radii['implicit-Euler'] = radius
    _, radius, exact_radius = run_imex_Euler(t0=t0, dt=dt, Tend=Tend)
    radii['imex-Euler'] = radius
    _, radius, exact_radius = run_CrankNicholson(t0=t0, dt=dt, Tend=Tend)
    radii['CrankNicholson'] = radius

    xcoords = [t0 + i * dt for i in range(int((Tend - t0) / dt))]
    plot_radius(xcoords, exact_radius, radii)

def main_error(cwd=''):

    t0 = 0
    Tend = 0.032

    errors = {}
    # err, _, _ = run_implicit_Euler(t0=t0, dt=0.001/512, Tend=Tend)
    # errors['implicit-Euler'] = err
    # err, _, _ = run_imex_Euler(t0=t0, dt=0.001/512, Tend=Tend)
    # errors['imex-Euler'] = err
    err, _, _ = run_CrankNicholson(t0=t0, dt=0.001/64, Tend=Tend)
    errors['CrankNicholson'] = err

if __name__ == "__main__":
    main_error()
    # main_radius()


"""
Several basic tests for hierarchical clustering procedures

"""
# Authors: Vincent Michel, 2010, Gael Varoquaux 2012
# License: BSD 3 clause
import warnings
from tempfile import mkdtemp

import numpy as np
from scipy import sparse
from scipy.cluster import hierarchy

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_almost_equal

from sklearn.cluster import Ward, WardAgglomeration, ward_tree
from sklearn.cluster.hierarchical import _hc_cut
from sklearn.feature_extraction.image import grid_to_graph

def test_structured_ward_tree():
    """
    Check that we obtain the correct solution for structured ward tree.
    """
    rnd = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    # Avoiding a mask with only 'True' entries
    mask[4:7, 4:7] = 0
    X = rnd.randn(50, 100)
    connectivity = grid_to_graph(*mask.shape)
    children, n_components, n_leaves, parent = ward_tree(X.T, connectivity)
    n_nodes = 2 * X.shape[1] - 1
    assert_true(len(children) + n_leaves == n_nodes)
    # Check that ward_tree raises a ValueError with a connectivity matrix
    # of the wrong shape
    assert_raises(ValueError, ward_tree, X.T, np.ones((4, 4)))

def test_unstructured_ward_tree():
    """
    Check that we obtain the correct solution for unstructured ward tree.
    """
    rnd = np.random.RandomState(0)
    X = rnd.randn(50, 100)
    for this_X in (X, X[0]):
        with warnings.catch_warnings(record=True) as warning_list:
            warnings.simplefilter("always", UserWarning)
            warnings.simplefilter("ignore", DeprecationWarning)
            # With specified a number of clusters just for the sake of
            # raising a warning and testing the warning code
            children, n_nodes, n_leaves, parent = ward_tree(this_X.T,
                                                            n_clusters=10)
        assert_equal(len(warning_list), 1)
        n_nodes = 2 * X.shape[1] - 1
        assert_equal(len(children) + n_leaves, n_nodes)

def test_height_ward_tree():
    """
    Check that the height of ward tree is sorted.
    """
    rnd = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    X = rnd.randn(50, 100)
    connectivity = grid_to_graph(*mask.shape)
    children, n_nodes, n_leaves, parent = ward_tree(X.T, connectivity)
    n_nodes = 2 * X.shape[1] - 1
    assert_true(len(children) + n_leaves == n_nodes)

def test_ward_clustering():
    """
    Check that we obtain the correct number of clusters with Ward clustering.
    """
    rnd = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    X = rnd.randn(100, 50)
    connectivity = grid_to_graph(*mask.shape)
    clustering = Ward(n_clusters=10, connectivity=connectivity)
    clustering.fit(X)
    # test caching
    clustering = Ward(n_clusters=10, connectivity=connectivity,
                      memory=mkdtemp())
    clustering.fit(X)
    labels = clustering.labels_
    assert_true(np.size(np.unique(labels)) == 10)
    # Turn caching off now
    clustering = Ward(n_clusters=10, connectivity=connectivity)
    # Check that we obtain the same solution with early-stopping of the
    # tree building
    clustering.compute_full_tree = False
    clustering.fit(X)
    np.testing.assert_array_equal(clustering.labels_, labels)
    clustering.connectivity = None
    clustering.fit(X)
    assert_true(np.size(np.unique(clustering.labels_)) == 10)
    # Check that we raise a TypeError on dense matrices
    clustering = Ward(n_clusters=10,
                      connectivity=connectivity.todense())
    assert_raises(TypeError, clustering.fit, X)
    clustering = Ward(n_clusters=10,
                      connectivity=sparse.lil_matrix(
                          connectivity.todense()[:10, :10]))
    assert_raises(ValueError, clustering.fit, X)

def test_ward_agglomeration():
    """
    Check that we obtain the correct solution in a simplistic case
    """
    rnd = np.random.RandomState(0)
    mask = np.ones([10, 10], dtype=np.bool)
    X = rnd.randn(50, 100)
    connectivity = grid_to_graph(*mask.shape)
    ward = WardAgglomeration(n_clusters=5, connectivity=connectivity)
    ward.fit(X)
    assert_true(np.size(np.unique(ward.labels_)) == 5)

    Xred = ward.transform(X)
    assert_true(Xred.shape[1] == 5)
    Xfull = ward.inverse_transform(Xred)
    assert_true(np.unique(Xfull[0]).size == 5)
    assert_array_almost_equal(ward.transform(Xfull), Xred)

def assess_same_labelling(cut1, cut2):
    """Util for comparison with scipy"""
    co_clust = []
    for cut in [cut1, cut2]:
        n = len(cut)
        k = cut.max() + 1
        ecut = np.zeros((n, k))
        ecut[np.arange(n), cut] = 1
        co_clust.append(np.dot(ecut, ecut.T))
    assert_true((co_clust[0] == co_clust[1]).all())

def test_scikit_vs_scipy():
    """Test scikit ward with full connectivity (i.e. unstructured) vs scipy
    """
    from scipy.sparse import lil_matrix
    n, p, k = 10, 5, 3
    rnd = np.random.RandomState(0)

    connectivity = lil_matrix(np.ones((n, n)))
    for i in range(5):
        X = .1 * rnd.normal(size=(n, p))
        X -= 4 * np.arange(n)[:, np.newaxis]
        X -= X.mean(axis=1)[:, np.newaxis]

        out = hierarchy.ward(X)

        children_ = out[:, :2].astype(np.int)
        children, _, n_leaves, _ = ward_tree(X, connectivity)

        cut = _hc_cut(k, children, n_leaves)
        cut_ = _hc_cut(k, children_, n_leaves)
        assess_same_labelling(cut, cut_)

    # Test error management in _hc_cut
    assert_raises(ValueError, _hc_cut, n_leaves + 1, children, n_leaves)

def test_connectivity_popagation():
    """
    Check that connectivity in the ward tree is propagated correctly during
    merging.
    """
    from sklearn.neighbors import NearestNeighbors

    X = np.array([(.014, .120), (.014, .099), (.014, .097),
                  (.017, .153), (.017, .153), (.018, .153),
                  (.018, .153), (.018, .153), (.018, .153),
                  (.018, .153), (.018, .153), (.018, .153),
                  (.018, .152), (.018, .149), (.018, .144),
                  ])
    nn = NearestNeighbors(n_neighbors=10).fit(X)
    connectivity = nn.kneighbors_graph(X)
    ward = Ward(n_clusters=4, connectivity=connectivity)
    # If changes are not propagated correctly, fit crashes with an
    # IndexError
    ward.fit(X)

def test_connectivity_fixing_non_lil():
    """
    Check non regression of a bug if a non item assignable connectivity is
    provided with more than one component.
    """
    # create dummy data
    x = np.array([[0, 0], [1, 1]])
    # create a mask with several components to force connectivity fixing
    m = np.array([[True, False], [False, True]])
    c = grid_to_graph(n_x=2, n_y=2, mask=m)
    w = Ward(connectivity=c)
    with warnings.catch_warnings(record=True):
        w.fit(x)

if __name__ == '__main__':
    import nose
    nose.run(argv=['', __file__])


# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import math
import os
import random
import zipfile

import numpy as np
from six.moves import urllib
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

# Step 1: Download the data.
url = 'http://mattmahoney.net/dc/'

def maybe_download(filename, expected_bytes):
  """Download a file if not present, and make sure it's the right size."""
  if not os.path.exists(filename):
    filename, _ = urllib.request.urlretrieve(url + filename, filename)
  statinfo = os.stat(filename)
  if statinfo.st_size == expected_bytes:
    print('Found and verified', filename)
  else:
    print(statinfo.st_size)
    raise Exception(
        'Failed to verify ' + filename + '. Can you get to it with a browser?')
  return filename

filename = maybe_download('text8.zip', 31344016)

# Read the data into a list of strings.
def read_data(filename):
  """Extract the first file enclosed in a zip file as a list of words"""
  with zipfile.ZipFile(filename) as f:
    data = tf.compat.as_str(f.read(f.namelist()[0])).split()
  return data

words = read_data(filename)
print('Data size', len(words))

# Step 2: Build the dictionary and replace rare words with UNK token.
vocabulary_size = 50000

def build_dataset(words):
  count = [['UNK', -1]]
  count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
  dictionary = dict()
  for word, _ in count:
    dictionary[word] = len(dictionary)
  data = list()
  unk_count = 0
  for word in words:
    if word in dictionary:
      index = dictionary[word]
    else:
      index = 0  # dictionary['UNK']
      unk_count += 1
    data.append(index)
  count[0][1] = unk_count
  reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
  return data, count, dictionary, reverse_dictionary

data, count, dictionary, reverse_dictionary = build_dataset(words)
del words  # Hint to reduce memory.
print('Most common words (+UNK)', count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])

data_index = 0

# Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
  global data_index
  assert batch_size % num_skips == 0
  assert num_skips <= 2 * skip_window
  batch = np.ndarray(shape=(batch_size), dtype=np.int32)
  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
  span = 2 * skip_window + 1 # [ skip_window target skip_window ]
  buffer = collections.deque(maxlen=span)
  for _ in range(span):
    buffer.append(data[data_index])
    data_index = (data_index + 1) % len(data)
  for i in range(batch_size // num_skips):
    target = skip_window  # target label at the center of the buffer
    targets_to_avoid = [ skip_window ]
    for j in range(num_skips):
      while target in targets_to_avoid:
        target = random.randint(0, span - 1)
      targets_to_avoid.append(target)
      batch[i * num_skips + j] = buffer[skip_window]
      labels[i * num_skips + j, 0] = buffer[target]
    buffer.append(data[data_index])
    data_index = (data_index + 1) % len(data)
  return batch, labels

batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
  print(batch[i], reverse_dictionary[batch[i]],
      '->', labels[i, 0], reverse_dictionary[labels[i, 0]])

# Step 4: Build and train a skip-gram model.

batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
skip_window = 1       # How many words to consider left and right.
num_skips = 2         # How many times to reuse an input to generate a label.

# We pick a random validation set to sample nearest neighbors. Here we limit the
# validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent.
valid_size = 16     # Random set of words to evaluate similarity on.
valid_window = 100  # Only pick dev samples in the head of the distribution.
valid_examples = np.random.choice(valid_window, valid_size, replace=False)
num_sampled = 64    # Number of negative examples to sample.

graph = tf.Graph()

with graph.as_default():

  # Input data.
  train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
  train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
  valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

  # Ops and variables pinned to the CPU because of missing GPU implementation
  with tf.device('/cpu:0'):
    # Look up embeddings for inputs.
    embeddings = tf.Variable(
        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
    embed = tf.nn.embedding_lookup(embeddings, train_inputs)

    # Construct the variables for the NCE loss
    nce_weights = tf.Variable(
        tf.truncated_normal([vocabulary_size, embedding_size],
                            stddev=1.0 / math.sqrt(embedding_size)))
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

  # Compute the average NCE loss for the batch.
  # tf.nce_loss automatically draws a new sample of the negative labels each
  # time we evaluate the loss.
  loss = tf.reduce_mean(
      tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels,
                     num_sampled, vocabulary_size))

  # Construct the SGD optimizer using a learning rate of 1.0.
  optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)

  # Compute the cosine similarity between minibatch examples and all embeddings.
  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
  normalized_embeddings = embeddings / norm
  valid_embeddings = tf.nn.embedding_lookup(
      normalized_embeddings, valid_dataset)
  similarity = tf.matmul(
      valid_embeddings, normalized_embeddings, transpose_b=True)

  # Add variable initializer.
  init = tf.initialize_all_variables()

# Step 5: Begin training.
num_steps = 100001

with tf.Session(graph=graph) as session:
  # We must initialize all variables before we use them.
  init.run()
  print("Initialized")

  average_loss = 0
  for step in xrange(num_steps):
    batch_inputs, batch_labels = generate_batch(
        batch_size, num_skips, skip_window)
    feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}

    # We perform one update step by evaluating the optimizer op (including it
    # in the list of returned values for session.run()
    _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
    average_loss += loss_val

    if step % 2000 == 0:
      if step > 0:
        average_loss /= 2000
      # The average loss is an estimate of the loss over the last 2000 batches.
      print("Average loss at step ", step, ": ", average_loss)
      average_loss = 0

    # Note that this is expensive (~20% slowdown if computed every 500 steps)
    if step % 10000 == 0:
      sim = similarity.eval()
      for i in xrange(valid_size):
        valid_word = reverse_dictionary[valid_examples[i]]
        top_k = 8 # number of nearest neighbors
        nearest = (-sim[i, :]).argsort()[1:top_k+1]
        log_str = "Nearest to %s:" % valid_word
        for k in xrange(top_k):
          close_word = reverse_dictionary[nearest[k]]
          log_str = "%s %s," % (log_str, close_word)
        print(log_str)
  final_embeddings = normalized_embeddings.eval()

# Step 6: Visualize the embeddings.

def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
  assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
  plt.figure(figsize=(18, 18))  #in inches
  for i, label in enumerate(labels):
    x, y = low_dim_embs[i,:]
    plt.scatter(x, y)
    plt.annotate(label,
                 xy=(x, y),
                 xytext=(5, 2),
                 textcoords='offset points',
                 ha='right',
                 va='bottom')

  plt.savefig(filename)

try:
  from sklearn.manifold import TSNE
  import matplotlib.pyplot as plt

  tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
  plot_only = 500
  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only,:])
  labels = [reverse_dictionary[i] for i in xrange(plot_only)]
  plot_with_labels(low_dim_embs, labels)

except ImportError:
  print("Please install sklearn, matplotlib, and scipy to visualize embeddings.")