import os
import re

import pandas as pd
import tensorflow as tf

pjoin = os.path.join
DATA_DIR = pjoin(os.path.dirname(__file__), 'data')

train_data = pd.read_csv(pjoin(DATA_DIR, 'train.csv'))
test_data = pd.read_csv(pjoin(DATA_DIR, 'test.csv'))

# Translation:
#  Don: an honorific title used in Spain, Portugal, Italy
#  Dona: Feminine form for don
#  Mme: Madame, Mrs
#  Mlle: Mademoiselle, Miss
#  Jonkheer (female equivalent: Jonkvrouw) is a Dutch honorific of nobility
HONORABLE_TITLES = ['sir', 'lady', 'don', 'dona', 'countess', 'jonkheer',
                    'major', 'col', 'dr', 'master', 'capt']
NORMAL_TITLES = ['mr', 'ms', 'mrs', 'miss', 'mme', 'mlle', 'rev']
TITLES = HONORABLE_TITLES + NORMAL_TITLES

def get_title(name):
    title_search = re.search('([A-Za-z]+)\.', name)
    return title_search.group(1).lower()

def get_family(row):
    last_name = row['Name'].split(",")[0]
    if last_name:
        family_size = 1 + row['Parch'] + row['SibSp']
        if family_size > 3:
            return "{0}_{1}".format(last_name.lower(), family_size)
        else:
            return "nofamily"
    else:
        return "unknown"

def get_deck(cabin):
    if pd.isnull(cabin):
        return 'U'
    return cabin[:1]

class TitanicDigest(object):
    def __init__(self, dataset):
        self.count_by_sex = dataset.groupby('Sex')['PassengerId'].count()
        self.mean_age = dataset['Age'].mean()
        self.mean_age_by_sex = dataset.groupby("Sex")["Age"].mean()
        self.mean_fare_by_class = dataset.groupby("Pclass")["Fare"].mean()
        self.titles = TITLES
        self.families = dataset.apply(get_family, axis=1).unique().tolist()
        self.decks = dataset["Cabin"].apply(get_deck).unique().tolist()
        self.embarkments = dataset.Embarked.unique().tolist()
        self.embark_mode = dataset.Embarked.dropna().mode().values

def preprocess(data, digest):
    # convert ['male', 'female'] values of Sex to [1, 0]
    data['Sex'] = data['Sex'].apply(lambda s: 1 if s == 'male' else 0)
    # fill empty age field with mean age
    data['Age'] = data['Age'].apply(
        lambda age: digest.mean_age if pd.isnull(age) else age)

    # is child flag
    data['Child'] = data['Age'].apply(lambda age: 1 if age <= 15 else 0)

    # fill fare with mean fare of the class
    def get_fare_value(row):
        if pd.isnull(row['Fare']):
            return digest.mean_fare_by_class[row['Pclass']]
        else:
            return row['Fare']

    data['Fare'] = data.apply(get_fare_value, axis=1)

    # fill Embarked with mode
    data['Embarked'] = data['Embarked'].apply(
        lambda e: digest.embark_mode if pd.isnull(e) else e)
    data["EmbarkedF"] = data["Embarked"].apply(digest.embarkments.index)

    #
    data['Cabin'] = data['Cabin'].apply(lambda c: 'U0' if pd.isnull(c) else c)

    # Deck
    data["Deck"] = data["Cabin"].apply(lambda cabin: cabin[0])
    data["DeckF"] = data['Deck'].apply(digest.decks.index)

    data['Title'] = data['Name'].apply(get_title)
    data['TitleF'] = data['Title'].apply(digest.titles.index)

    data['Honor'] = data['Title'].apply(
        lambda title: int(title in HONORABLE_TITLES))

    data['Family'] = data.apply(get_family, axis=1)

    if 'Survived' in data.keys():
        data['Deceased'] = data['Survived'].apply(lambda s: int(not s))
    return data

digest = TitanicDigest(train_data)

def get_train_data():
    return preprocess(train_data, digest)

def get_test_data():
    return preprocess(test_data, digest)

def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def transform_to_tfrecord():
    data = pd.read_csv(pjoin(DATA_DIR, 'train.csv'))
    filepath = pjoin(DATA_DIR, 'data.tfrecords')
    writer = tf.python_io.TFRecordWriter(filepath)
    for i in range(len(data)):
        feature = {}
        for key in data.keys():
            value = data[key][i]
            if isinstance(value, int):
                value = tf.train.Feature(
                    int64_list=tf.train.Int64List(value=[value]))
            elif isinstance(value, float):
                value = tf.train.Feature(
                    float_list=tf.train.FloatList(value=[value])
                )
            elif isinstance(value, str):
                value = tf.train.Feature(
                    bytes_list=tf.train.BytesList(
                        value=[value.encode(encoding="utf-8")])
                )
            feature[key] = value
        example = tf.train.Example(
            features=tf.train.Features(feature=feature))
        writer.write(example.SerializeToString())
    writer.close()

if __name__ == '__main__':
    transform_to_tfrecord()


import matplotlib.pyplot as plt
import matplotlib.patches as patches

def route(root):

	root_height = root[2]

	coordinates = [\
[0.42*root_height+root[0],0.42*root_height+root[1],root_height/2],\
[-0.42*root_height+root[0],0.42*root_height+root[1],root_height/2],\
[-0.42*root_height+root[0],-0.15*root_height+root[1],root_height/2],\
[0.42*root_height+root[0],-0.15*root_height+root[1],root_height/2]]

	return coordinates

if __name__ == "__main__":

	meter_to_feet = 3.28
	root = [0,0,16*1]
	print 'root',root,'\n'

	level1 = route(root)
	print 'level 1 \n'
	print level1[0],'\n'
	print level1[1],'\n'
	print level1[2],'\n'
	print level1[3],'\n'

	print 'level 2 \n'
	level2 = [[0]*3]*4

	for x in xrange(4):
		level2[x] = route(level1[x])
		for y in xrange(4):
			print 'level2 point[',x+1,y+1,']',level2[x][y],'\n'

	fig, ax = plt.subplots()

	ball, = plt.plot(6.72+1.52,6.72+1.52,'mo')

	plt.plot(0,0,'bo')
	plt.plot([level1[0][0],level1[1][0],level1[2][0],level1[3][0]],[level1[0][1],level1[1][1],level1[2][1],level1[3][1]],'ro')

	rect_blue = patches.Rectangle((-13.44,-4.8),13.44*2,9.12*2,linewidth=1,edgecolor='b',facecolor='b',alpha = 0.1)
	ax.add_patch(rect_blue)

	rect_red = patches.Rectangle((0,4.23),13.44,9.12,linewidth=1,edgecolor='r',facecolor='r',alpha = 0.3)
	ax.add_patch(rect_red)

	plt.plot([level2[0][0][0],level2[0][1][0],level2[0][2][0],level2[0][3][0]],[level2[0][0][1],level2[0][1][1],level2[0][2][1],level2[0][3][1]],'go')

	rect_green = patches.Rectangle((6.72,6.72+4.23/2),13.44/2,9.12/2,linewidth=1,edgecolor='g',facecolor='g',alpha = 0.5)
	ax.add_patch(rect_green)

	linear_s = [12,12]
	plt.plot(12,12,'yo')
	rect_yellow = patches.Rectangle((10,11),13.44/4,9.12/4,linewidth=1,edgecolor='y',facecolor='y',alpha = 0.5)
	ax.add_patch(rect_yellow)

	ax.legend([ball,rect_blue,rect_red,rect_green,rect_yellow],['Ball','Root View','Level 1 - 4 anchors','Level 2 - 16 anchors','Linear Search - 64 anchors'])

	plt.axis([-13.44, 13.44, -4.8, 13.44])
	plt.show()


"""
Tests for DBSCAN clustering algorithm
"""

import pickle

import numpy as np

from scipy.spatial import distance
from scipy import sparse

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_in
from sklearn.utils.testing import assert_not_in
from sklearn.neighbors import NearestNeighbors
from sklearn.cluster.dbscan_ import DBSCAN
from sklearn.cluster.dbscan_ import dbscan
from sklearn.cluster.tests.common import generate_clustered_data
from sklearn.metrics.pairwise import pairwise_distances

n_clusters = 3
X = generate_clustered_data(n_clusters=n_clusters)

def test_dbscan_similarity():
    # Tests the DBSCAN algorithm with a similarity array.
    # Parameters chosen specifically for this task.
    eps = 0.15
    min_samples = 10
    # Compute similarities
    D = distance.squareform(distance.pdist(X))
    D /= np.max(D)
    # Compute DBSCAN
    core_samples, labels = dbscan(D, metric="precomputed", eps=eps,
                                  min_samples=min_samples)
    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - (1 if -1 in labels else 0)

    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric="precomputed", eps=eps, min_samples=min_samples)
    labels = db.fit(D).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

def test_dbscan_feature():
    # Tests the DBSCAN algorithm with a feature vector array.
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    metric = 'euclidean'
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(X, metric=metric, eps=eps,
                                  min_samples=min_samples)

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples)
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

def test_dbscan_sparse():
    core_sparse, labels_sparse = dbscan(sparse.lil_matrix(X), eps=.8,
                                        min_samples=10)
    core_dense, labels_dense = dbscan(X, eps=.8, min_samples=10)
    assert_array_equal(core_dense, core_sparse)
    assert_array_equal(labels_dense, labels_sparse)

def test_dbscan_sparse_precomputed():
    D = pairwise_distances(X)
    nn = NearestNeighbors(radius=.9).fit(X)
    D_sparse = nn.radius_neighbors_graph(mode='distance')
    # Ensure it is sparse not merely on diagonals:
    assert D_sparse.nnz < D.shape[0] * (D.shape[0] - 1)
    core_sparse, labels_sparse = dbscan(D_sparse,
                                        eps=.8,
                                        min_samples=10,
                                        metric='precomputed')
    core_dense, labels_dense = dbscan(D, eps=.8, min_samples=10,
                                      metric='precomputed')
    assert_array_equal(core_dense, core_sparse)
    assert_array_equal(labels_dense, labels_sparse)

def test_dbscan_no_core_samples():
    rng = np.random.RandomState(0)
    X = rng.rand(40, 10)
    X[X < .8] = 0

    for X_ in [X, sparse.csr_matrix(X)]:
        db = DBSCAN(min_samples=6).fit(X_)
        assert_array_equal(db.components_, np.empty((0, X_.shape[1])))
        assert_array_equal(db.labels_, -1)
        assert_equal(db.core_sample_indices_.shape, (0,))

def test_dbscan_callable():
    # Tests the DBSCAN algorithm with a callable metric.
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    # metric is the function reference, not the string key.
    metric = distance.euclidean
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(X, metric=metric, eps=eps,
                                  min_samples=min_samples,
                                  algorithm='ball_tree')

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples,
                algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

def test_dbscan_balltree():
    # Tests the DBSCAN algorithm with balltree for neighbor calculation.
    eps = 0.8
    min_samples = 10

    D = pairwise_distances(X)
    core_samples, labels = dbscan(D, metric="precomputed", eps=eps,
                                  min_samples=min_samples)

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_1, n_clusters)

    db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_2, n_clusters)

    db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='kd_tree')
    labels = db.fit(X).labels_

    n_clusters_3 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_3, n_clusters)

    db = DBSCAN(p=1.0, eps=eps, min_samples=min_samples, algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_4 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_4, n_clusters)

    db = DBSCAN(leaf_size=20, eps=eps, min_samples=min_samples,
                algorithm='ball_tree')
    labels = db.fit(X).labels_

    n_clusters_5 = len(set(labels)) - int(-1 in labels)
    assert_equal(n_clusters_5, n_clusters)

def test_input_validation():
    # DBSCAN.fit should accept a list of lists.
    X = [[1., 2.], [3., 4.]]
    DBSCAN().fit(X)             # must not raise exception

def test_dbscan_badargs():
    # Test bad argument values: these should all raise ValueErrors
    assert_raises(ValueError,
                  dbscan,
                  X, eps=-1.0)
    assert_raises(ValueError,
                  dbscan,
                  X, algorithm='blah')
    assert_raises(ValueError,
                  dbscan,
                  X, metric='blah')
    assert_raises(ValueError,
                  dbscan,
                  X, leaf_size=-1)
    assert_raises(ValueError,
                  dbscan,
                  X, p=-1)

def test_pickle():
    obj = DBSCAN()
    s = pickle.dumps(obj)
    assert_equal(type(pickle.loads(s)), obj.__class__)

def test_boundaries():
    # ensure min_samples is inclusive of core point
    core, _ = dbscan([[0], [1]], eps=2, min_samples=2)
    assert_in(0, core)
    # ensure eps is inclusive of circumference
    core, _ = dbscan([[0], [1], [1]], eps=1, min_samples=2)
    assert_in(0, core)
    core, _ = dbscan([[0], [1], [1]], eps=.99, min_samples=2)
    assert_not_in(0, core)

def test_weighted_dbscan():
    # ensure sample_weight is validated
    assert_raises(ValueError, dbscan, [[0], [1]], sample_weight=[2])
    assert_raises(ValueError, dbscan, [[0], [1]], sample_weight=[2, 3, 4])

    # ensure sample_weight has an effect
    assert_array_equal([], dbscan([[0], [1]], sample_weight=None,
                                  min_samples=6)[0])
    assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 5],
                                  min_samples=6)[0])
    assert_array_equal([0], dbscan([[0], [1]], sample_weight=[6, 5],
                                   min_samples=6)[0])
    assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[6, 6],
                                      min_samples=6)[0])

    # points within eps of each other:
    assert_array_equal([0, 1], dbscan([[0], [1]], eps=1.5,
                                      sample_weight=[5, 1], min_samples=6)[0])
    # and effect of non-positive and non-integer sample_weight:
    assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 0],
                                  eps=1.5, min_samples=6)[0])
    assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[5.9, 0.1],
                                      eps=1.5, min_samples=6)[0])
    assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[6, 0],
                                      eps=1.5, min_samples=6)[0])
    assert_array_equal([], dbscan([[0], [1]], sample_weight=[6, -1],
                                  eps=1.5, min_samples=6)[0])

    # for non-negative sample_weight, cores should be identical to repetition
    rng = np.random.RandomState(42)
    sample_weight = rng.randint(0, 5, X.shape[0])
    core1, label1 = dbscan(X, sample_weight=sample_weight)
    assert_equal(len(label1), len(X))

    X_repeated = np.repeat(X, sample_weight, axis=0)
    core_repeated, label_repeated = dbscan(X_repeated)
    core_repeated_mask = np.zeros(X_repeated.shape[0], dtype=bool)
    core_repeated_mask[core_repeated] = True
    core_mask = np.zeros(X.shape[0], dtype=bool)
    core_mask[core1] = True
    assert_array_equal(np.repeat(core_mask, sample_weight), core_repeated_mask)

    # sample_weight should work with precomputed distance matrix
    D = pairwise_distances(X)
    core3, label3 = dbscan(D, sample_weight=sample_weight,
                           metric='precomputed')
    assert_array_equal(core1, core3)
    assert_array_equal(label1, label3)

    # sample_weight should work with estimator
    est = DBSCAN().fit(X, sample_weight=sample_weight)
    core4 = est.core_sample_indices_
    label4 = est.labels_
    assert_array_equal(core1, core4)
    assert_array_equal(label1, label4)

    est = DBSCAN()
    label5 = est.fit_predict(X, sample_weight=sample_weight)
    core5 = est.core_sample_indices_
    assert_array_equal(core1, core5)
    assert_array_equal(label1, label5)
    assert_array_equal(label1, est.labels_)

def test_dbscan_core_samples_toy():
    X = [[0], [2], [3], [4], [6], [8], [10]]
    n_samples = len(X)

    for algorithm in ['brute', 'kd_tree', 'ball_tree']:
        # Degenerate case: every sample is a core sample, either with its own
        # cluster or including other close core samples.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=1)
        assert_array_equal(core_samples, np.arange(n_samples))
        assert_array_equal(labels, [0, 1, 1, 1, 2, 3, 4])

        # With eps=1 and min_samples=2 only the 3 samples from the denser area
        # are core samples. All other points are isolated and considered noise.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=2)
        assert_array_equal(core_samples, [1, 2, 3])
        assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1])

        # Only the sample in the middle of the dense area is core. Its two
        # neighbors are edge samples. Remaining samples are noise.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=3)
        assert_array_equal(core_samples, [2])
        assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1])

        # It's no longer possible to extract core samples with eps=1:
        # everything is noise.
        core_samples, labels = dbscan(X, algorithm=algorithm, eps=1,
                                      min_samples=4)
        assert_array_equal(core_samples, [])
        assert_array_equal(labels, -np.ones(n_samples))

def test_dbscan_precomputed_metric_with_degenerate_input_arrays():
    # see https://github.com/scikit-learn/scikit-learn/issues/4641 for
    # more details
    X = np.eye(10)
    labels = DBSCAN(eps=0.5, metric='precomputed').fit(X).labels_
    assert_equal(len(set(labels)), 1)

    X = np.zeros((10, 10))
    labels = DBSCAN(eps=0.5, metric='precomputed').fit(X).labels_
    assert_equal(len(set(labels)), 1)


#!/usr/local/bin/python
# -*- coding: utf-8 -*-

import calendar
import os

import pandas as pd

# Правила подсчета очков:
#
# за участие в матче – 2 очка, если сыграно 10 минут и больше; 1 очко, если сыграно меньше 10 минут
#
# за победу – 3 очка (в гостях); 2 очка (дома)
#
# за поражение – минус 3 очка (дома); минуc 2 очка (в гостях)
#
# Принцип начисления очков следующий:
#
# количество очков + количество передач + количество перехватов + количество подборов +
# количество блок-шотов + количество совершенных штрафных + количество совершенных двухочковых + количество совершенных трехочковых
#
#  - количество попыток штрафных бросков - количество попыток двухочковых – количество попыток трехочковых –
# удвоенная цифра от количества потерь - количество фолов

def convert_to_sec(time_str):
    if pd.isnull(time_str):
        return 0
    try:
        m, s = time_str.split(':')
        return int(m) * 60 + int(s)
    except Exception as inst:
        print(time_str)
        print(type(inst))  # the exception instance
        print(inst.args)  # arguments stored in .args

def get_sec(row):
    time_str = row['minutes']
    return convert_to_sec(time_str)

def getOrDefault(value):
    if pd.isnull(value):
        return 0
    return int(value)

def extractYear(row):
    return row['date'].year

def extractMonth(row):
    return row['date'].month

def extractDay(row):
    return row['date'].day

def concat1(row):
    return row['opponent'] + str(row['year'])

def concat2(row):
    return row['team'] + str(row['year'])

def concat3(row):
    return row['name'] + str(row['year'])

def concat4(row):
    return row['opponent'] + str(row['month']) + str(row['year'])

def concat5(row):
    return row['team'] + str(row['month']) + str(row['year'])

def concat6(row):
    return row['name'] + str(row['month']) + str(row['year'])

def getDayOfTheWeek(row):
    day = calendar.day_name[row['date'].weekday()]
    return day[:3]

def convert_age(row):
    if pd.isnull(row['age']):
        return 0
    years, days = row['age'].split('-')
    return int(years) + 1.0 * int(days) / 365

def split_result(x):
    try:
        sp = x.split('(')
        return sp[0].strip(), sp[1][:-1]
    except Exception as inst:
        print(x)
        print(type(inst))  # the exception instance
        print(inst.args)  # arguments stored in .args

#   (FG + 0.5 * 3P) / FGA
def calc_efg(row):
    try:
        fg = row['fg']
        fg3 = row['fg3']
        fga = row['fga']
        if fga == 0:
            return 0.0
        return (fg + 0.5 * fg3) / fga
    except Exception as inst:
        print(row)
        print(type(inst))  # the exception instance
        print(inst.args)  # arguments stored in .args

def calc_fantasy(row):
    if pd.isnull(row['minutes']):
        return 0
    fantasy_points = 0
    if convert_to_sec(row['minutes']) >= 10 * 60:
        fantasy_points += 2
    else:
        fantasy_points += 1
    if 'W' in str(row['result']):
        if row['location'] == '@':
            fantasy_points += 3
        else:
            fantasy_points += 2
    else:
        if row['location'] == '@':
            fantasy_points -= 2
        else:
            fantasy_points -= 3
    fantasy_points += getOrDefault(row['pts'])
    fantasy_points += getOrDefault(row['ast'])
    fantasy_points += getOrDefault(row['stl'])
    fantasy_points += getOrDefault(row['trb'])
    fantasy_points += getOrDefault(row['blk'])
    fantasy_points += getOrDefault(row['ft'])
    fantasy_points += getOrDefault(row['fg'])

    fantasy_points -= getOrDefault(row['fta'])
    fantasy_points -= getOrDefault(row['fga'])
    fantasy_points -= 2 * getOrDefault(row['tov'])
    fantasy_points -= getOrDefault(row['pf'])

    return fantasy_points

def enrich_player_df(df):
    df['fantasy_points'] = df.apply(lambda row: calc_fantasy(row), axis=1)
    df['year'] = df.apply(lambda row: extractYear(row), axis=1)
    df['month'] = df.apply(lambda row: extractMonth(row), axis=1)
    df['day'] = df.apply(lambda row: extractDay(row), axis=1)
    df['opponent2'] = df.apply(lambda row: concat1(row), axis=1)
    df['opponent3'] = df.apply(lambda row: concat4(row), axis=1)
    df['team3'] = df.apply(lambda row: concat5(row), axis=1)
    df['name3'] = df.apply(lambda row: concat6(row), axis=1)
    df['name2'] = df.apply(lambda row: concat3(row), axis=1)
    df['team2'] = df.apply(lambda row: concat2(row), axis=1)
    df['age1'] = df.apply(lambda row: convert_age(row), axis=1)
    df['seconds'] = df.apply(lambda row: get_sec(row), axis=1)

    for i in range(1, 6):
        df['mean_pts_' + str(i)] = df['pts'].rolling(i).mean().shift(1)

    df['efg'] = df.apply(lambda row: calc_efg(row), axis=1)
    df['mefg'] = df['efg'].expanding().mean().shift(1)
    df['day_of_the_week'] = df.apply(lambda row: getDayOfTheWeek(row), axis=1)
    df['mfp'] = df['fantasy_points'].expanding().mean().shift(1)
    df['medfp'] = df['fantasy_points'].expanding().median().shift(1)
    df['msec'] = df['seconds'].expanding().mean().shift(1)
    df['mpts'] = df['pts'].expanding().mean().shift(1)
    df['mast'] = df['ast'].expanding().mean().shift(1)
    df['mtrb'] = df['trb'].expanding().mean().shift(1)
    df['mstl'] = df['stl'].expanding().mean().shift(1)
    df['mpf'] = df['pf'].expanding().mean().shift(1)
    df['mtov'] = df['tov'].expanding().mean().shift(1)
    df['mblk'] = df['blk'].expanding().mean().shift(1)
    df['mfg'] = df['fg'].expanding().mean().shift(1)
    df['mfg3'] = df['fg3'].expanding().mean().shift(1)
    df['mft'] = df['ft'].expanding().mean().shift(1)

    df['mfg3_pct'] = df['fg3_pct'].expanding().mean().shift(1)
    df['mfg_pct'] = df['fg_pct'].expanding().mean().shift(1)
    df['mft_pct'] = df['ft_pct'].expanding().mean().shift(1)

    # number of games in last 5 days
    df['rest_days'] = df['date'].diff().apply(lambda x: x.days)

    for i in [1, 7, 10, 11, 12]:
        df['mean_rest_days_' + str(i)] = df['rest_days'].rolling(i).mean().shift(1)

    for i in [10, 21, 31, 38, 39]:
        df['mean_fantasy_' + str(i)] = df['fantasy_points'].rolling(i).mean().shift(1)

    for i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:
        df['mean_sec_' + str(i)] = df['seconds'].rolling(i).mean().shift(1)

    for i in [3, 4, 12, 16, 17, 28, 36]:
        df['skew_fantasy_' + str(i)] = df['fantasy_points'].rolling(i).skew().shift(1)

    return df

def enrich_player_df_for_upcoming_games(df):
    df['fantasy_points'] = df.apply(lambda row: calc_fantasy(row), axis=1)
    df['year'] = df.apply(lambda row: extractYear(row), axis=1)
    df['month'] = df.apply(lambda row: extractMonth(row), axis=1)
    df['day'] = df.apply(lambda row: extractDay(row), axis=1)
    df['opponent2'] = df.apply(lambda row: concat1(row), axis=1)
    df['opponent3'] = df.apply(lambda row: concat4(row), axis=1)
    df['team3'] = df.apply(lambda row: concat5(row), axis=1)
    df['name3'] = df.apply(lambda row: concat6(row), axis=1)
    df['name2'] = df.apply(lambda row: concat3(row), axis=1)
    df['team2'] = df.apply(lambda row: concat2(row), axis=1)
    df['age1'] = df.apply(lambda row: convert_age(row), axis=1)
    df['seconds'] = df.apply(lambda row: get_sec(row), axis=1)

    for i in range(1, 6):
        df['mean_pts_' + str(i)] = df['pts'].rolling(i).mean().shift(1)

    df['efg'] = df.apply(lambda row: calc_efg(row), axis=1)
    df['mefg'] = df['efg'].expanding().mean().shift(1)
    df['day_of_the_week'] = df.apply(lambda row: getDayOfTheWeek(row), axis=1)
    df['mfp'] = df['fantasy_points'].expanding().mean().shift(1)
    df['medfp'] = df['fantasy_points'].expanding().median().shift(1)
    df['msec'] = df['seconds'].expanding().mean().shift(1)
    df['mpts'] = df['pts'].expanding().mean().shift(1)
    df['mast'] = df['ast'].expanding().mean().shift(1)
    df['mtrb'] = df['trb'].expanding().mean().shift(1)
    df['mstl'] = df['stl'].expanding().mean().shift(1)
    df['mpf'] = df['pf'].expanding().mean().shift(1)
    df['mtov'] = df['tov'].expanding().mean().shift(1)
    df['mblk'] = df['blk'].expanding().mean().shift(1)
    df['mfg'] = df['fg'].expanding().mean().shift(1)
    df['mfg3'] = df['fg3'].expanding().mean().shift(1)
    df['mft'] = df['ft'].expanding().mean().shift(1)

    df['mfg3_pct'] = df['fg3_pct'].expanding().mean().shift(1)
    df['mfg_pct'] = df['fg_pct'].expanding().mean().shift(1)
    df['mft_pct'] = df['ft_pct'].expanding().mean().shift(1)

    # number of games in last 5 days
    df['rest_days'] = df['date'].diff().apply(lambda x: x.days)

    for i in [1, 7, 10, 11, 12]:
        df['mean_rest_days_' + str(i)] = df['rest_days'].rolling(i).mean().shift(1)

    for i in [10, 21, 31, 38, 39]:
        df['mean_fantasy_' + str(i)] = df['fantasy_points'].rolling(i).mean().shift(1)

    for i in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:
        df['mean_sec_' + str(i)] = df['seconds'].rolling(i).mean().shift(1)

    for i in [3, 4, 12, 16, 17, 28, 36]:
        df['skew_fantasy_' + str(i)] = df['fantasy_points'].rolling(i).skew().shift(1)
    return df

dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d')

def enrich_all_data():
    for root, dirs, files in os.walk("nba"):
        for file in files:
            if file.endswith(".csv"):
                try:
                    path = os.path.join(root, file)
                    if path.find('fantasy') == -1 and path.find('2018.csv') != -1:
                        f = open(path)
                        print(path)
                        lines = f.readlines()
                        if len(lines) > 1:
                            df = pd.read_csv(path,
                                             parse_dates=['date'],
                                             date_parser=dateparse)
                        if not df.empty:
                            df.fillna(df.mean(), inplace=True)
                            df = enrich_player_df(df)
                            join = os.path.join(root, "fantasy")
                            if not os.path.exists(join):
                                os.mkdir(join)
                            df.to_csv(os.path.join(root, "fantasy", file), index=False)
                except Exception as inst:
                    print(file)
                    print(df.head())
                    print(type(inst))  # the exception instance
                    print(inst.args)  # arguments stored in .args


import matplotlib.pyplot as plt
import scipy.signal
import numpy as np
import math
import random
from matplotlib.backends.backend_pdf import PdfPages

samples = 51200
L = samples/512
fs = 512e6
dt = 1/fs
time = [i*dt for i in range(samples)]

def pfb_fir(x):
	N = len(x)
	T = 4
	L = 512
	bin_width_scale = 2.5
	dx = T*math.pi/L/T
	X = np.array([n*dx-T*math.pi/2 for n in range(T*L)])
	coeff = np.sinc(bin_width_scale*X/math.pi)*np.hanning(T*L)

	y = np.array([0+0j]*(N-T*L))
	for n in range((T-1)*L, N):
		m = n%L
		coeff_sub = coeff[L*T-m::-L]
		y[n-T*L] = (x[n-(T-1)*L:n+L:L]*coeff_sub).sum()

	return y

R = 100/5
#freqs = [i*1e5 + 6.0e6 for i in range(R)]
freqs = [i*5e4 + 6.0e6 for i in range(R*8)]

bin = []
bin_pfb = []
for f in freqs:
	print f
	signal = np.array([complex(math.cos(2*math.pi*f*t), math.sin(2*math.pi*f*t)) for t in time])
	y = pfb_fir(signal)

	bin_pfb.append(np.fft.fft(y[0:512])[10])

bin = np.array(bin)
bin_pfb = np.array(bin_pfb)
freqs = np.array(freqs)/1e6

b = scipy.signal.firwin(20, cutoff=0.125, window="hanning")
w,h = scipy.signal.freqz(b,1, 4*R, whole=1)
h = np.array(h[2*R:4*R].tolist()+h[0:2*R].tolist())
#h = np.array(h[20:40].tolist()+h[0:20].tolist())

fig = plt.figure()
ax0 = fig.add_subplot(111)

#ax0.plot(freqs, abs(fir9), '.', freqs, abs(fir10), '.', freqs, abs(fir11), '.')

ax0.plot(freqs, 10*np.log10(abs(bin_pfb)/512), 'k-')

ax0.set_xlabel('Frequency (MHz)')
ax0.set_ylabel('Gain (dB)')
ax0.set_ylim((-50,0))
plt.show()
#ax0.axvline(x = 10, linewidth=1, color='k')


"""Voxel-Based Morphometry on Oasis dataset
========================================

This example uses Voxel-Based Morphometry (VBM) to study the relationship
between aging, sex and gray matter density.

The data come from the `OASIS <http://www.oasis-brains.org/>`_ project.
If you use it, you need to agree with the data usage agreement available
on the website.

It has been run through a standard VBM pipeline (using SPM8 and
NewSegment) to create VBM maps, which we study here.

VBM analysis of aging
---------------------

We run a standard GLM analysis to study the association between age
and gray matter density from the VBM data. We use only 100 subjects
from the OASIS dataset to limit the memory usage.

Note that more power would be obtained from using a larger sample of subjects.
"""
# Authors: Bertrand Thirion, <bertrand.thirion@inria.fr>, July 2018
#          Elvis Dhomatob, <elvis.dohmatob@inria.fr>, Apr. 2014
#          Virgile Fritsch, <virgile.fritsch@inria.fr>, Apr 2014
#          Gael Varoquaux, Apr 2014

n_subjects = 100  # more subjects requires more memory

############################################################################
# Load Oasis dataset
# ------------------

from nilearn import datasets
oasis_dataset = datasets.fetch_oasis_vbm(n_subjects=n_subjects)
gray_matter_map_filenames = oasis_dataset.gray_matter_maps
age = oasis_dataset.ext_vars['age'].astype(float)

###############################################################################
# Sex is encoded as 'M' or 'F'. Hence, we make it a binary variable.
sex = oasis_dataset.ext_vars['mf'] == b'F'

###############################################################################
# Print basic information on the dataset.
print('First gray-matter anatomy image (3D) is located at: %s' %
      oasis_dataset.gray_matter_maps[0])  # 3D data
print('First white-matter anatomy image (3D) is located at: %s' %
      oasis_dataset.white_matter_maps[0])  # 3D data

###############################################################################
# Get a mask image: A mask of the  cortex of the ICBM template.
gm_mask = datasets.fetch_icbm152_brain_gm_mask()

###############################################################################
# Resample the images, since this mask has a different resolution.
from nilearn.image import resample_to_img
mask_img = resample_to_img(
    gm_mask, gray_matter_map_filenames[0], interpolation='nearest')

#############################################################################
# Analyse data
# ------------
#
# First, we create an adequate design matrix with three columns: 'age',
# 'sex', 'intercept'.
import pandas as pd
import numpy as np
intercept = np.ones(n_subjects)
design_matrix = pd.DataFrame(np.vstack((age, sex, intercept)).T,
                             columns=['age', 'sex', 'intercept'])

#############################################################################
# Let's plot the design matrix.
from nistats.reporting import plot_design_matrix

ax = plot_design_matrix(design_matrix)
ax.set_title('Second level design matrix', fontsize=12)
ax.set_ylabel('maps')

##########################################################################
# Next, we specify and fit the second-level model when loading the data and also
# smooth a little bit to improve statistical behavior.

from nistats.second_level_model import SecondLevelModel
second_level_model = SecondLevelModel(smoothing_fwhm=2.0, mask_img=mask_img)
second_level_model.fit(gray_matter_map_filenames,
                       design_matrix=design_matrix)

##########################################################################
# Estimating the contrast is very simple. We can just provide the column
# name of the design matrix.
z_map = second_level_model.compute_contrast(second_level_contrast=[1, 0, 0],
                                            output_type='z_score')

###########################################################################
# We threshold the second level contrast at uncorrected p < 0.001 and plot it.
from nistats.thresholding import map_threshold
from nilearn import plotting

_, threshold = map_threshold(
    z_map, alpha=.05, height_control='fdr')
print('The FDR=.05-corrected threshold is: %.3g' % threshold)

display = plotting.plot_stat_map(
    z_map, threshold=threshold, colorbar=True, display_mode='z',
    cut_coords=[-4, 26],
    title='age effect on grey matter density (FDR = .05)')
plotting.show()

###########################################################################
# We can also study the effect of sex by computing the contrast, thresholding it
# and plot the resulting map.

z_map = second_level_model.compute_contrast(second_level_contrast='sex',
                                            output_type='z_score')
_, threshold = map_threshold(
    z_map, alpha=.05, height_control='fdr')
plotting.plot_stat_map(
    z_map, threshold=threshold, colorbar=True,
    title='sex effect on grey matter density (FDR = .05)')

###########################################################################
# Note that there does not seem to be any significant effect of sex on
# grey matter density on that dataset.

###########################################################################
# Generating a report
# -------------------
# It can be useful to quickly generate a
# portable, ready-to-view report with most of the pertinent information.
# This is easy to do if you have a fitted model and the list of contrasts,
# which we do here.

from nistats.reporting import make_glm_report

icbm152_2009 = datasets.fetch_icbm152_2009()
report = make_glm_report(model=second_level_model,
                         contrasts=['age', 'sex'],
                         bg_img=icbm152_2009['t1'],
                         )

#########################################################################
# We have several ways to access the report:

# report  # This report can be viewed in a notebook
# report.save_as_html('report.html')
# report.open_in_browser()


#from dask.distributed import Client
import xarray as xr
import numpy as np
import pandas as pd

import importlib
import ELMlib
importlib.reload(ELMlib)

#client = Client(n_workers=2, threads_per_worker=2, memory_limit='1GB')
#client

#ds = xr.open_dataset('../Data/14C_spinup_holger_fire.2x2_small.nc')
from netCDF4 import Dataset
ds = Dataset('../Data/14C_spinup_holger_fire.2x2_small.nc')

#lat, lon = ds.coords['lat'], ds.coords['lon']
lat, lon = ds['lat'][:], ds['lon'][:]
lat_indices, lon_indices = np.meshgrid(
    range(len(lat)),
    range(len(lon)),
    indexing='ij'
)

lats, lons = np.meshgrid(lat, lon, indexing='ij')
df_pd = pd.DataFrame(
    {
        'cell_nr': range(len(lat)*len(lon)),
        'lat_index': lat_indices.flatten(),
        'lon_index': lon_indices.flatten(),
        'lat': lats.flatten(),
        'lon': lons.flatten()
    }
)

import dask.array as da

import dask.dataframe as dask_df

df_dask = dask_df.from_pandas(df_pd, npartitions=4)
df_dask

parameter_set = ELMlib.load_parameter_set(
    ds_filename = '../Data/14C_spinup_holger_fire.2x2_small.nc',
    time_shift  = -198*365,
    nstep       = 10
)

def func(line):
    location_dict = {
        'cell_nr':   int(line.cell_nr),
        'lat_index': int(line.lat_index),
        'lon_index': int(line.lon_index)
    }

    cell_nr, log, xs_12C_data, us_12C_data, rs_12C_data= ELMlib.load_model_12C_data(parameter_set, location_dict)
    return cell_nr, log, xs_12C_data, us_12C_data, rs_12C_data

df_dask_2 = df_dask.apply(func, axis=1, meta=('A', 'object'))

df_dask_2.compute()
type(df_dask_2)

df_dask_2

list(df_dask_2)

pd.DataFrame(list(df_dask_2), columns=('cell_nr', 'log', 'xs_12C_data', 'us_12C_data', 'rs_12C_data'))


"""Utilities for meta-estimators"""
# Author: Joel Nothman
#         Andreas Mueller
# Licence: BSD

from operator import attrgetter
from functools import update_wrapper

__all__ = ['if_delegate_has_method']

class _IffHasAttrDescriptor(object):
    """Implements a conditional property using the descriptor protocol.

    Using this class to create a decorator will raise an ``AttributeError``
    if the ``attribute_name`` is not present on the base object.

    This allows ducktyping of the decorated method based on ``attribute_name``.

    See https://docs.python.org/3/howto/descriptor.html for an explanation of
    descriptors.
    """
    def __init__(self, fn, attribute_name):
        self.fn = fn
        self.get_attribute = attrgetter(attribute_name)
        # update the docstring of the descriptor
        update_wrapper(self, fn)

    def __get__(self, obj, type=None):
        # raise an AttributeError if the attribute is not present on the object
        if obj is not None:
            # delegate only on instances, not the classes.
            # this is to allow access to the docstrings.
            self.get_attribute(obj)
        # lambda, but not partial, allows help() to work with update_wrapper
        out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)
        # update the docstring of the returned function
        update_wrapper(out, self.fn)
        return out

def if_delegate_has_method(delegate):
    """Create a decorator for methods that are delegated to a sub-estimator

    This enables ducktyping by hasattr returning True according to the
    sub-estimator.

    >>> from sklearn.utils.metaestimators import if_delegate_has_method
    >>>
    >>>
    >>> class MetaEst(object):
    ...     def __init__(self, sub_est):
    ...         self.sub_est = sub_est
    ...
    ...     @if_delegate_has_method(delegate='sub_est')
    ...     def predict(self, X):
    ...         return self.sub_est.predict(X)
    ...
    >>> class HasPredict(object):
    ...     def predict(self, X):
    ...         return X.sum(axis=1)
    ...
    >>> class HasNoPredict(object):
    ...     pass
    ...
    >>> hasattr(MetaEst(HasPredict()), 'predict')
    True
    >>> hasattr(MetaEst(HasNoPredict()), 'predict')
    False
    """
    return lambda fn: _IffHasAttrDescriptor(fn, '%s.%s' % (delegate, fn.__name__))


from collections import OrderedDict

import pandas as pd
import pandas.util.testing as tm
import pytest

import ibis
import ibis.expr.datatypes as dt

@pytest.fixture(scope="module")
def value():
    return OrderedDict([("fruit", "pear"), ("weight", 0)])

@pytest.fixture(scope="module")
def struct_client(value):
    df = pd.DataFrame(
        {
            "s": [
                OrderedDict([("fruit", "apple"), ("weight", None)]),
                value,
                OrderedDict([("fruit", "pear"), ("weight", 1)]),
            ],
            "key": list("aab"),
            "value": [1, 2, 3],
        }
    )
    return ibis.pandas.connect({"t": df})

@pytest.fixture
def struct_table(struct_client):
    return struct_client.table(
        "t",
        schema={
            "s": dt.Struct.from_tuples(
                [("fruit", dt.string), ("weight", dt.int8)]
            )
        },
    )

def test_struct_field_literal(value):
    struct = ibis.literal(value)
    assert struct.type() == dt.Struct.from_tuples(
        [("fruit", dt.string), ("weight", dt.int8)]
    )

    expr = struct.fruit
    result = ibis.pandas.execute(expr)
    assert result == "pear"

    expr = struct.weight
    result = ibis.pandas.execute(expr)
    assert result == 0

def test_struct_field_series(struct_table):
    t = struct_table
    expr = t.s.fruit
    result = expr.execute()
    expected = pd.Series(["apple", "pear", "pear"], name="fruit")
    tm.assert_series_equal(result, expected)

def test_struct_field_series_group_by_key(struct_table):
    t = struct_table
    expr = t.groupby(t.s.fruit).aggregate(total=t.value.sum())
    result = expr.execute()
    expected = pd.DataFrame(
        [("apple", 1), ("pear", 5)], columns=["fruit", "total"]
    )
    tm.assert_frame_equal(result, expected)

def test_struct_field_series_group_by_value(struct_table):
    t = struct_table
    expr = t.groupby(t.key).aggregate(total=t.s.weight.sum())
    result = expr.execute()
    # these are floats because we have a NULL value in the input data
    expected = pd.DataFrame([("a", 0.0), ("b", 1.0)], columns=["key", "total"])
    tm.assert_frame_equal(result, expected)


from common2 import *

# NAME IDEA -> pooling/random/sparse/distributed hebbian/horde/crowd/fragment/sample memory

# FEATURES:
# + boost -- neurons with empty mem slots learn faster
# + noise --
# + dropout -- temporal disabling of neurons
# + decay -- remove from mem
# + negatives -- learning to avoid detecting some patterns
# + fatigue -- winner has lower score for some time
# - sklearn -- compatibile api
# - prune -- if input < mem shrink mem ? (problem with m > input len

# IDEA:
# - popularity -- most popular neuron is cloned / killed

# NEXT VERSION:
# - layers -- rsm stacking

# NEXT VERSION:
# - attention
#   - https://towardsdatascience.com/the-fall-of-rnn-lstm-2d1594c74ce0
#   - https://towardsdatascience.com/memory-attention-sequences-37456d271992

# NEXT VERSION:
# - numpy -- faster version
# - cython -- faster version
# - gpu -- faster version
# - distributed

class rsm:
	def __init__(self,n,m):
		"""Random Sample Memory
			n -- number of neurons
			m -- max connections per neuron (memory)
		"""
		self.N = n
		self.M = m
		self.mem = {j:set() for j in range(n)}
		self.win = {j:0 for j in range(n)}
		self.tow = {j:-42000 for j in range(n)} # time of win
		self.t = 0

	# ---[ core ]---------------------------------------------------------------

	# TODO -- input length vs mem length
	def scores(self, input, boost=False, noise=False, fatigue=0, dropout=0.0): # -> dict[i] -> scores
		"""
			input -- sparse binary features
			boost -- improve scores based on number of unconnected synapses (TODO)
			noise -- randomize scores to prevent snowballing
			dropout -- temporal disabling of neurons
		"""
		mem = self.mem
		tow = self.tow
		N = self.N
		M = self.M
		t = self.t
		scores = {}
		for j in mem:
			scores[j] = len(input & mem[j])
		if noise:
			for j in mem:
				scores[j] += 0.9*random()
		if boost:
			for j in mem:
				scores[j] += 1+2*(M-len(mem[j])) if len(mem[j])<M else 0
		if fatigue:
			for j in mem:
				dt = 1.0*min(fatigue,t - tow[j])
				factor = dt / fatigue
				scores[j] *= factor
		if dropout:
			k = int(round(float(dropout)*N))
			for j in combinations(N,k):
				scores[j] = -1
		return scores

	def learn(self, input, k, decay=0.0, dropout=0.0, fatigue=0,
	          negative=False, boost=True, noise=True):
		"""
			input -- sparse binary features
			k -- number of winning neurons
		"""
		mem = self.mem
		win = self.win
		tow = self.tow
		M = self.M
		t = self.t

		known_inputs = set()
		for j in mem:
			known_inputs.update(mem[j])

		scores = self.scores(input, boost=boost, noise=noise, dropout=dropout, fatigue=fatigue)
		winners = top(k,scores)
		for j in winners:

			# negative learning
			if negative:
				mem[j].difference_update(input)
				continue

			# positive learning
			unknown_inputs = input - known_inputs
			mem[j].update(pick(unknown_inputs, M-len(mem[j])))
			known_inputs.update(mem[j])

			# handle decay
			if decay:
				decay_candidates = mem[j] - input
				if decay_candidates:
					for d in decay_candidates:
						if random() < decay:
							mem[j].remove(d)

			# handle popularity
			win[j] += 1

			# handle fatigue
			tow[j] = t

		self.t += 1

	# ---[ auxiliary ]----------------------------------------------------------

	def fit(self, X, Y):
		for x,y in zip (X,Y):
			negative = not y
			self.learn(x,negative=negative)

	def score_many(self, X, k=1, method=1):
		out = []
		for x in X:
			s = self.score_one(x,k,method)
			out += [s]
		return out

	def transform(self, X, k=1, method=1, cutoff=0.5):
		out = []
		for s in self.score_many(X,k,method):
			y = 1 if s>=cutoff else 0
			out += [y]
		return out

	def confusion(self, X, Y, k=1, method=1, cutoff=0.5):
		PY = self.transform(X,k,method,cutoff)
		p = 0
		n = 0
		tp = 0
		tn = 0
		fp = 0
		fn = 0
		for y,py in zip(Y,PY):
			if y:  p+=1
			else:  n+=1

			if y:
				if py: tp+=1
				else:  fn+=1
			else:
				if py: fp+=1
				else:  tn+=1
		return dict(p=p,n=n,tp=tp,tn=tn,fp=fp,fn=fn)

	def score(self, X, Y, k=1, method=1, cutoff=0.5, kind='acc'):
		c = self.confusion(X,Y,k,method,cutoff)
		p = float(c['p'])
		n = float(c['n'])
		tp = float(c['tp'])
		tn = float(c['tn'])
		fp = float(c['fp'])
		fn = float(c['fn'])
		if kind=='f1':
			return (2*tp) / (2*tp + fp + fn)
		elif kind=='acc':
			return (tp+tn) / (p+n)
		elif kind=='prec':
			return tp / (tp + fp)
		elif kind=='sens':
			return tp / (tp + fn)
		elif kind=='spec':
			return tn / (tn + fp)

	def score_one(self, input, k=1, method=1):
		"aggregate scores to scalar"
		scores = self.scores(input)
		if method==0:
			return top(k, scores, values=True)
		elif method==1:
			score = 1.0*sum(top(k, scores, values=True))/(k*(self.M+1))
			return score
		elif method==2:
			score = 1.0*sum(top(k, scores, values=True))/(k*self.M)
			return min(1.0,score)
		if method==3:
			score = 1.0*min(top(k, scores, values=True))/(self.M+1)
			return score
		elif method==4:
			score = 1.0*min(top(k, scores, values=True))/self.M
			return min(1.0,score)
		if method==5:
			score = 1.0*max(top(k, scores, values=True))/(self.M+1)
			return score
		elif method==6:
			score = 1.0*max(top(k, scores, values=True))/self.M
			return min(1.0,score)

	def stats(self,prefix=''):
		vol_v = self.vol.values()
		mem_v = self.mem.values()
		out = {}
		out['m_empty']     = sum([1.0 if len(x)==0 else 0.0 for x in mem_v])/self.N
		out['m_not_empty'] = sum([1.0 if len(x)>0 else 0.0 for x in mem_v])/self.N
		out['m_full']      = sum([1.0 if len(x)==self.M else 0.0 for x in mem_v])/self.N
		out['m_avg']       = sum([1.0*len(x) for x in mem_v])/(self.N*self.M)
		return {k:v for k,v in out.items() if k.startswith(prefix)}


""" Module storing classifiers for YATSM

Contains utilities and helper classes for classifying timeseries generated
using YATSM change detection.
"""
import logging

from sklearn.ensemble import RandomForestClassifier
import yaml

from ..errors import AlgorithmNotFoundException

logger = logging.getLogger('yatsm')

_algorithms = {
    'RandomForest': RandomForestClassifier
}

def cfg_to_algorithm(config_file):
    """ Return instance of classification algorithm helper from config file

    Args:
        config_file (str): location of configuration file for algorithm

    Returns:
        tuple: scikit-learn estimator (object) and configuration file (dict)

    Raises:
        KeyError: raise if configuration file is malformed
        AlgorithmNotFoundException: raise if algorithm is not implemented in
            YATSM
        TypeError: raise if configuration file cannot be used to initialize
            the classifier

    """
    # Determine which algorithm is used
    try:
        with open(config_file, 'r') as f:
            config = yaml.safe_load(f)
    except Exception as e:
        logger.error('Could not read config file {} ({})'
                     .format(config_file, str(e)))
        raise

    algo_name = config['algorithm']
    if algo_name not in _algorithms.keys():
        raise AlgorithmNotFoundException(
            'Could not process unknown algorithm named "%s"' % algo_name)
    else:
        algo = _algorithms[algo_name]

    if algo_name not in config:
        logger.warning('%s algorithm parameters not found in config file %s. '
                       'Using default values.' % (algo_name, config_file))
        config[algo_name] = {}

    # Try to load algorithm using hyperparameters from config
    try:
        sklearn_algo = algo(**config[algo_name].get('init', {}))
    except TypeError:
        logger.error('Cannot initialize %s classifier. Config file %s '
                     'contains unknown options' % (algo_name, config_file))
        raise

    return sklearn_algo, config