# -*- coding: utf-8 -*-

import urllib.parse
import urllib.request
import posixpath
import datetime
import os
import logging
import filecmp
import difflib
import json
import sqlite3
import hashlib
import yaml
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
#from bokeh.io import output_notebook
# output_notebook()

# Logging Setup
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    datefmt='%d %b %Y %H:%M:%S')
logger = logging.getLogger()

# create download and output folders if they do not exist
os.makedirs(os.path.join('download'), exist_ok=True)
os.makedirs(os.path.join('output'), exist_ok=True)
os.makedirs(os.path.join('output', 'original_data'), exist_ok=True)

def get_sha_hash(path, blocksize=65536):
    sha_hasher = hashlib.sha256()
    with open(path, 'rb') as f:
        buffer = f.read(blocksize)
        while len(buffer) > 0:
            sha_hasher.update(buffer)
            buffer = f.read(blocksize)
        return sha_hasher.hexdigest()

def downloadandcache(url):
    """
    Download a file into a folder called "downloads".
    The file is prefixed with the download date YYYY-M-D-.
    Returns the local filepath.

    Parameters
    ----------
    url : str
        Url of a file to be downloaded

    """

    path = urllib.parse.urlsplit(url).path
    filename = posixpath.basename(path)
    now = datetime.datetime.now()
    datestring = str(now.year) + "-" + str(now.month) + "-" + str(now.day)
    filepath = os.path.join('download', datestring + "-" + filename)
    filepath_original_data = os.path.join('output',
                                          'original_data',
                                          filename)

    # check if file exists, otherwise download it
    if not os.path.exists(filepath):
        logger.info('Downloading file %s', filename)
        urllib.request.urlretrieve(url, filepath)
        urllib.request.urlretrieve(url, filepath_original_data)
    else:
        logger.info('Using local file from %s', filepath)

    foldername = 'download'

    return foldername, datestring, filename

def decrementday(year, month, day):
    """
    Given values for year, month, and day, the values of the previous day are
    returned. At the moment, the function assumes that every month has 31 days,
    so that it will return February 31st when given values for March 1.

    Parameters
    ----------
    year : integer
        Integer year
    month : integer
        Integer month
    day : integer
        Integer day
    """

    if day > 1:
        day = day - 1
    else:
        day = 31
        if month > 1:
            month = month - 1
        else:
            month = 12
            year = year - 1

    return year, month, day

def getolderfilenameandcleanup(foldername, datestring, filename):
    """
    Given a set of foldername and filename as returned by
    the downloadandcache function, an older non-identical file with
    the same file structure is searched in the folder. Files identical to the
    one given are deleted.

    Parameters
    ----------
    foldername : str
        folder where files are located
    datestring : str
        string of original file date YYYY-M-D
    filename : str
        filename of original file

    """
    originalfilepath = os.path.join(foldername, datestring + "-" + filename)
    now = datetime.datetime.now()
    year = now.year
    month = now.month
    day = now.day
    # loop through older possible files
    i = 0
    while i < 2000:
        i = i + 1
        year, month, day = decrementday(year, month, day)
        datestring = str(year) + "-" + str(month) + "-" + str(day)
        filepath = os.path.join(foldername, datestring + "-" + filename)
        # Does the file exist?
        if os.path.isfile(filepath):
            # Check if file is identical to original file. If yes delete this
            # file and continue
            if filecmp.cmp(originalfilepath, filepath):
                # print('files are identical, deleting', filepath)
                os.remove(filepath)
            else:
                # print('files are not identical:', filepath)
                return filepath
    raise ValueError('no older file found')

def getmatchinglist():
    """
    This function returns the matchinglist located under
    /input/matching_bnetza_uba.csv

    Parameters
    ----------
    none
    """
    # read matching list
    result = pd.read_csv(
        os.path.join('input/data/DE', 'matching_bnetza_uba.csv'),
        skiprows=0,
        sep=',',  # CSV field separator, default is ','
        thousands=',',  # Thousands separator, default is ','
        decimal='.',  # Decimal separator, default is '.')
        encoding='cp1252')
    result['uba_id_string'] = (result['uba_match_name'] + '_'
                               + result['uba_match_fuel'])
    return result

def getbnetzalist(url_bnetza, previous=False):
    """
    This function returns the dataframe of the plantlist by the
    Bundesnetzagentur. if previous == True, the next-oldest different plantlist
    in the folder is returned as determined by the function
    getolderfilenameandcleanup.

    Parameters
    ----------
    url_bnetza : str
        URL of plant list
    previous : boolean
        Should previous plant list be returned?

    """
    foldername, datestring, filename = downloadandcache(url_bnetza)
    if not previous:
        plantlist = pd.read_csv(os.path.join(foldername, datestring + "-" + filename),
                                skiprows=9,
                                sep=';',  # CSV field separator, default is ','
                                thousands='.',  # Thousands separator, default is ','
                                decimal=',',  # Decimal separator, default is '.'
                                encoding='cp1252')
        return plantlist
    elif previous:
        oldfilename = getolderfilenameandcleanup(foldername, datestring, filename)
        oldplantlist = pd.read_csv(oldfilename,
                                   skiprows=9,
                                   sep=';',  # CSV field separator, default is ','
                                   thousands='.',  # Thousands separator, default is ','
                                   decimal=',',  # Decimal separator, default is '.'
                                   encoding='cp1252')
        return oldplantlist

def getubalist(url_uba, previous=False):
    """
    This function returns the dataframe of the plantlist by the
    Umweltbundesamt. if previous == True, the next-oldest different plantlist
    in the folder is returned as determined by the function
    getolderfilenameandcleanup.

    Parameters
    ----------
    url_uba : str
        URL of plant list
    previous : boolean
        Should previous plant list be returned?

    """
    foldername, datestring, filename = downloadandcache(url_uba)
    if not previous:
        plantlist = pd.read_excel(os.path.join(foldername, datestring + "-" + filename), skiprows=9)
        return plantlist
    elif previous:
        oldfilename = getolderfilenameandcleanup(foldername, datestring, filename)
        oldplantlist = pd.read_excel(oldfilename, skiprows=9)
        return oldplantlist

def getlistdifferences(oldplantlist, newplantlist):
    """
    This function returns the difference between two plantlists, and only takes
    into account the columns specified within the function.

    Parameters
    ----------
    oldplantlist : DataFrame
        Old Plantlist
    newplantlist : DataFrame
        New Plantlist

    """
    oldplantlist['source'] = 'old'
    newplantlist['source'] = 'new'
    comparisonplantlist = pd.concat([oldplantlist, newplantlist])

    # Only include some columns in comparison
    includecolumns = ['Kraftwerksnummer Bundesnetzagentur',
                      'Kraftwerksname',
                      'Blockname',
                      'Kraftwerksname / Standort',
                      'Kraftwerksstandort',
                      'Primärenergieträger',
                     ]
    cols = [col for col in comparisonplantlist.columns if col in includecolumns]
    comparisonplantlist = comparisonplantlist.drop_duplicates(keep=False, subset=cols)
    # Sort by first column
    comparisonplantlist = comparisonplantlist.sort_values(comparisonplantlist.columns[0], ascending=True)
    return comparisonplantlist

def matchinglistcheck(url_bnetza, url_uba):
    """
    This function checks the BNetzA and UBA plantlists against the
    matchinglist and prints out errors. For entries form the UBA Plantlist a
    suggestion for correction with the closest possible match is printed.

    Parameters
    ----------
    oldplantlist : DataFrame
        Old Plantlist
    newplantlist : DataFrame
        New Plantlist

    """
    logger.info('Starting Matchinglistcheck')

    plantlist_uba = getubalist(url_uba)
    plantlist_bnetza = getbnetzalist(url_bnetza)
    matchinglist = getmatchinglist()

    plantlist_uba['uba_id_string'] = (plantlist_uba['Kraftwerksname / Standort']
                                      + '_' + plantlist_uba['Primärenergieträger'])
#    print(plantlist_uba.uba_id_string)
    matchinglist.rename(columns={'ID BNetzA': 'bnetza_id'}, inplace=True)

    uba_entrylist = [x for x in plantlist_uba.uba_id_string.tolist() if str(x) != 'nan']

    errorfound = False
    for entry in matchinglist.index:
        #        print(entry, matchinglist.loc[entry].bnetza_id,  matchinglist.loc[entry].uba_id_string)
        bnetza_entries = plantlist_bnetza.loc[(plantlist_bnetza['Kraftwerksnummer Bundesnetzagentur'] == matchinglist.loc[entry].bnetza_id)]
#        print(entry, len(bnetza_entries))
        if len(bnetza_entries) == 0:
            logger.error('Entry not in Bnetzalist:', matchinglist.loc[entry].bnetza_id, matchinglist.loc[entry].uba_id_string)
            errorfound = True
        uba_entries = plantlist_uba.loc[(plantlist_uba['uba_id_string'] == matchinglist.loc[entry].uba_id_string)]
#        print(entry, len(uba_entries))
        if len(uba_entries) == 0:
            alternatives = difflib.get_close_matches(matchinglist.loc[entry].uba_id_string, uba_entrylist, n=3, cutoff=0.6)
            logger.error('Not in ubalist: ' + matchinglist.loc[entry].uba_id_string + ' ' + matchinglist.loc[entry].bnetza_id + ' Possible alternatives: ' + ', '.join(alternatives))
#            raise ValueError('Value in Ubalist missing')
            errorfound = True
    if errorfound == False:
        logger.info('No obvious errors in Matchinglist check found')
    else:
        logger.error('Errors in Matchinglist exist')

def potentialmatching(url_bnetza, url_uba):
    """
    This function looks for power plants form the UBA list not contained in the
    matching lists. It looks up possible matches based on name similarity.
    It returns a list of tuples with the plants name of the UBA List, augmented
    with possible matches.

    Parameters
    ----------
    url_bnetza : string
        Link to BNetzA List
    url_uba: string
        Link to UBA List

    """
    plantlist_uba = getubalist(url_uba)
    plantlist_bnetza = getbnetzalist(url_bnetza)
    matchinglist = getmatchinglist()

    plantlist_bnetza.rename(columns={'Kraftwerksnummer Bundesnetzagentur':'id'}, inplace=True)
    plantlist_bnetza_reduced = plantlist_bnetza[plantlist_bnetza['id'].isin(matchinglist['ID BNetzA']) == False]
    plantlist_bnetza_reduced = plantlist_bnetza_reduced[plantlist_bnetza_reduced['Energieträger'] != 'Solare Strahlungsenergie']
    plantlist_bnetza_reduced = plantlist_bnetza_reduced[plantlist_bnetza_reduced['Energieträger'] != 'Windenergie (Onshore-Anlage)']
    plantlist_bnetza_reduced = plantlist_bnetza_reduced[plantlist_bnetza_reduced['Energieträger'] != 'Windenergie (Offshore-Anlage)']
    plantlist_bnetza_reduced['name_and_block'] = plantlist_bnetza_reduced['Kraftwerksname'] + ' ' + plantlist_bnetza_reduced['Blockname'] + '_' + plantlist_bnetza_reduced['Energieträger']

    plantlist_uba.rename(columns={'Kraftwerksname / Standort' : 'name',
                                  'Primärenergieträger': 'fuel',
                                  'Anlagenart': 'type'}, inplace=True)
#    print(plantlist_uba.columns)
    plantlist_uba['uba_id_string'] = (plantlist_uba['name']
                                      + '_' + plantlist_uba['fuel'])

    # Reduce uba list
    plantlist_uba_reduced = plantlist_uba[plantlist_uba['uba_id_string'].isin(matchinglist['uba_id_string']) == False]
    plantlist_uba_reduced = plantlist_uba_reduced[plantlist_uba_reduced['type'] != 'WEA']
    plantlist_uba_reduced = plantlist_uba_reduced[plantlist_uba_reduced['type'] != 'PV']
    plantlist_uba_reduced = plantlist_uba_reduced[plantlist_uba_reduced['type'].isnull() == False]

    possiblematcheslist = []
    for entry in plantlist_uba_reduced.index:
#        print(entry)
        moin = str(plantlist_uba_reduced.loc[entry].uba_id_string)
        moin2 = [x for x in plantlist_bnetza_reduced.name_and_block.tolist() if str(x) != 'nan']# plantlist_bnetza_reduced['name_and_block'].tolist()
#        print(moin)
#        print(moin2)
        possiblealternative = difflib.get_close_matches(moin, moin2, n=2, cutoff=0.3)
#        print(moin, possiblealternative)
        logger.info('Plant ' + moin + ' not in Matchinglist. Possible Matches from BNetzA List: ' + str(possiblealternative))
        possiblematcheslist.append((moin, possiblealternative))
#    return possiblematcheslist
    return plantlist_bnetza_reduced

# Testing this file
if __name__ == "__main__":

    # BNetzA Power plant list
    url_bnetza = ('http://www.bundesnetzagentur.de/SharedDocs/Downloads/DE/'
                  'Sachgebiete/Energie/Unternehmen_Institutionen/Versorgungssicherheit/'
                  'Erzeugungskapazitaeten/Kraftwerksliste/Kraftwerksliste_CSV.csv'
                  '?__blob=publicationFile&v=10')

    # UBA Power plant list
    url_uba = ('https://www.umweltbundesamt.de/sites/default/files/medien/372/dokumente/kraftwerke-de-ab-100-mw.xls')

    matchinglist = getmatchinglist()

    plantlist_bnetza = getbnetzalist(url_bnetza, previous=False)
#    plantlist_bnetza_previous = getbnetzalist(url_bnetza, previous=True)
#    plantlist_bnetza_differences = getlistdifferences(plantlist_bnetza_previous, plantlist_bnetza)

#    plantlist_uba = getubalist(url_uba, previous=False)
#    plantlist_uba_previous = getubalist(url_uba, previous=True)
#    plantlist_uba_differences = getlistdifferences(plantlist_uba_previous, plantlist_uba)

    matchinglistcheck(url_bnetza, url_uba)

    res = potentialmatching(url_bnetza, url_uba)


import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt

import nose.tools as nt
import numpy.testing as npt
from numpy.testing.decorators import skipif

from . import PlotTestCase
from .. import distributions as dist

try:
    import statsmodels.nonparametric.api
    assert statsmodels.nonparametric.api
    _no_statsmodels = False
except ImportError:
    _no_statsmodels = True

class TestKDE(PlotTestCase):

    rs = np.random.RandomState(0)
    x = rs.randn(50)
    y = rs.randn(50)
    kernel = "gau"
    bw = "scott"
    gridsize = 128
    clip = (-np.inf, np.inf)
    cut = 3

    def test_scipy_univariate_kde(self):
        """Test the univariate KDE estimation with scipy."""
        grid, y = dist._scipy_univariate_kde(self.x, self.bw, self.gridsize,
                                             self.cut, self.clip)
        nt.assert_equal(len(grid), self.gridsize)
        nt.assert_equal(len(y), self.gridsize)
        for bw in ["silverman", .2]:
            dist._scipy_univariate_kde(self.x, bw, self.gridsize,
                                       self.cut, self.clip)

    @skipif(_no_statsmodels)
    def test_statsmodels_univariate_kde(self):
        """Test the univariate KDE estimation with statsmodels."""
        grid, y = dist._statsmodels_univariate_kde(self.x, self.kernel,
                                                   self.bw, self.gridsize,
                                                   self.cut, self.clip)
        nt.assert_equal(len(grid), self.gridsize)
        nt.assert_equal(len(y), self.gridsize)
        for bw in ["silverman", .2]:
            dist._statsmodels_univariate_kde(self.x, self.kernel, bw,
                                             self.gridsize, self.cut,
                                             self.clip)

    def test_scipy_bivariate_kde(self):
        """Test the bivariate KDE estimation with scipy."""
        clip = [self.clip, self.clip]
        x, y, z = dist._scipy_bivariate_kde(self.x, self.y, self.bw,
                                            self.gridsize, self.cut, clip)
        nt.assert_equal(x.shape, (self.gridsize, self.gridsize))
        nt.assert_equal(y.shape, (self.gridsize, self.gridsize))
        nt.assert_equal(len(z), self.gridsize)

        # Test a specific bandwidth
        clip = [self.clip, self.clip]
        x, y, z = dist._scipy_bivariate_kde(self.x, self.y, 1,
                                            self.gridsize, self.cut, clip)

        # Test that we get an error with an invalid bandwidth
        with nt.assert_raises(ValueError):
            dist._scipy_bivariate_kde(self.x, self.y, (1, 2),
                                      self.gridsize, self.cut, clip)

    @skipif(_no_statsmodels)
    def test_statsmodels_bivariate_kde(self):
        """Test the bivariate KDE estimation with statsmodels."""
        clip = [self.clip, self.clip]
        x, y, z = dist._statsmodels_bivariate_kde(self.x, self.y, self.bw,
                                                  self.gridsize,
                                                  self.cut, clip)
        nt.assert_equal(x.shape, (self.gridsize, self.gridsize))
        nt.assert_equal(y.shape, (self.gridsize, self.gridsize))
        nt.assert_equal(len(z), self.gridsize)

    @skipif(_no_statsmodels)
    def test_statsmodels_kde_cumulative(self):
        """Test computation of cumulative KDE."""
        grid, y = dist._statsmodels_univariate_kde(self.x, self.kernel,
                                                   self.bw, self.gridsize,
                                                   self.cut, self.clip,
                                                   cumulative=True)
        nt.assert_equal(len(grid), self.gridsize)
        nt.assert_equal(len(y), self.gridsize)
        # make sure y is monotonically increasing
        npt.assert_((np.diff(y) > 0).all())

    def test_kde_cummulative_2d(self):
        """Check error if args indicate bivariate KDE and cumulative."""
        with npt.assert_raises(TypeError):
            dist.kdeplot(self.x, data2=self.y, cumulative=True)

    def test_bivariate_kde_series(self):
        df = pd.DataFrame({'x': self.x, 'y': self.y})

        ax_series = dist.kdeplot(df.x, df.y)
        ax_values = dist.kdeplot(df.x.values, df.y.values)

        nt.assert_equal(len(ax_series.collections),
                        len(ax_values.collections))
        nt.assert_equal(ax_series.collections[0].get_paths(),
                        ax_values.collections[0].get_paths())

class TestJointPlot(PlotTestCase):

    rs = np.random.RandomState(sum(map(ord, "jointplot")))
    x = rs.randn(100)
    y = rs.randn(100)
    data = pd.DataFrame(dict(x=x, y=y))

    def test_scatter(self):

        g = dist.jointplot("x", "y", self.data)
        nt.assert_equal(len(g.ax_joint.collections), 1)

        x, y = g.ax_joint.collections[0].get_offsets().T
        npt.assert_array_equal(self.x, x)
        npt.assert_array_equal(self.y, y)

        x_bins = dist._freedman_diaconis_bins(self.x)
        nt.assert_equal(len(g.ax_marg_x.patches), x_bins)

        y_bins = dist._freedman_diaconis_bins(self.y)
        nt.assert_equal(len(g.ax_marg_y.patches), y_bins)

    def test_reg(self):

        g = dist.jointplot("x", "y", self.data, kind="reg")
        nt.assert_equal(len(g.ax_joint.collections), 2)

        x, y = g.ax_joint.collections[0].get_offsets().T
        npt.assert_array_equal(self.x, x)
        npt.assert_array_equal(self.y, y)

        x_bins = dist._freedman_diaconis_bins(self.x)
        nt.assert_equal(len(g.ax_marg_x.patches), x_bins)

        y_bins = dist._freedman_diaconis_bins(self.y)
        nt.assert_equal(len(g.ax_marg_y.patches), y_bins)

        nt.assert_equal(len(g.ax_joint.lines), 1)
        nt.assert_equal(len(g.ax_marg_x.lines), 1)
        nt.assert_equal(len(g.ax_marg_y.lines), 1)

    def test_resid(self):

        g = dist.jointplot("x", "y", self.data, kind="resid")
        nt.assert_equal(len(g.ax_joint.collections), 1)
        nt.assert_equal(len(g.ax_joint.lines), 1)
        nt.assert_equal(len(g.ax_marg_x.lines), 0)
        nt.assert_equal(len(g.ax_marg_y.lines), 1)

    def test_hex(self):

        g = dist.jointplot("x", "y", self.data, kind="hex")
        nt.assert_equal(len(g.ax_joint.collections), 1)

        x_bins = dist._freedman_diaconis_bins(self.x)
        nt.assert_equal(len(g.ax_marg_x.patches), x_bins)

        y_bins = dist._freedman_diaconis_bins(self.y)
        nt.assert_equal(len(g.ax_marg_y.patches), y_bins)

    def test_kde(self):

        g = dist.jointplot("x", "y", self.data, kind="kde")

        nt.assert_true(len(g.ax_joint.collections) > 0)
        nt.assert_equal(len(g.ax_marg_x.collections), 1)
        nt.assert_equal(len(g.ax_marg_y.collections), 1)

        nt.assert_equal(len(g.ax_marg_x.lines), 1)
        nt.assert_equal(len(g.ax_marg_y.lines), 1)

    def test_color(self):

        g = dist.jointplot("x", "y", self.data, color="purple")

        purple = mpl.colors.colorConverter.to_rgb("purple")
        scatter_color = g.ax_joint.collections[0].get_facecolor()[0, :3]
        nt.assert_equal(tuple(scatter_color), purple)

        hist_color = g.ax_marg_x.patches[0].get_facecolor()[:3]
        nt.assert_equal(hist_color, purple)

    def test_annotation(self):

        g = dist.jointplot("x", "y", self.data)
        nt.assert_equal(len(g.ax_joint.legend_.get_texts()), 1)

        g = dist.jointplot("x", "y", self.data, stat_func=None)
        nt.assert_is(g.ax_joint.legend_, None)

    def test_hex_customise(self):

        # test that default gridsize can be overridden
        g = dist.jointplot("x", "y", self.data, kind="hex",
                           joint_kws=dict(gridsize=5))
        nt.assert_equal(len(g.ax_joint.collections), 1)
        a = g.ax_joint.collections[0].get_array()
        nt.assert_equal(28, a.shape[0])  # 28 hexagons expected for gridsize 5

    def test_bad_kind(self):

        with nt.assert_raises(ValueError):
            dist.jointplot("x", "y", self.data, kind="not_a_kind")


#!/usr/bin/python

import matplotlib.pyplot as plt
from prep_terrain_data import makeTerrainData
from class_vis import prettyPicture
from time import time
features_train, labels_train, features_test, labels_test = makeTerrainData()

### the training data (features_train, labels_train) have both "fast" and "slow" points mixed
### in together--separate them so we can give them different colors in the scatterplot,
### and visually identify them
grade_fast = [features_train[ii][0] for ii in range(0, len(features_train)) if labels_train[ii]==0]
bumpy_fast = [features_train[ii][1] for ii in range(0, len(features_train)) if labels_train[ii]==0]
grade_slow = [features_train[ii][0] for ii in range(0, len(features_train)) if labels_train[ii]==1]
bumpy_slow = [features_train[ii][1] for ii in range(0, len(features_train)) if labels_train[ii]==1]

#### initial visualization
plt.xlim(0.0, 1.0)
plt.ylim(0.0, 1.0)
plt.scatter(bumpy_fast, grade_fast, color = "b", label="fast")
plt.scatter(grade_slow, bumpy_slow, color = "r", label="slow")
plt.legend()
plt.xlabel("bumpiness")
plt.ylabel("grade")
plt.show()
#################################################################################

### your code here!  name your classifier object clf if you want the
### visualization code (prettyPicture) to show you the decision boundary

# K Nearest Neighbor
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score
clf = KNeighborsClassifier(n_neighbors=1)

t0 = time()
clf.fit(features_train, labels_train)
print "training time:", round(time()-t0, 3), "s"

t0 = time()
pred = clf.predict(features_test)
print "predicting time:", round(time()-t0, 3), "s"
acc = accuracy_score(pred, labels_test)
print acc

# Random Forest
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
clf = RandomForestClassifier(n_estimators=10)

t0 = time()
clf.fit(features_train, labels_train)
print "training time:", round(time()-t0, 3), "s"

t0 = time()
pred = clf.predict(features_test)
print "predicting time:", round(time()-t0, 3), "s"
acc = accuracy_score(pred, labels_test)
print acc

# Addaboost
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import accuracy_score
clf = AdaBoostClassifier(n_estimators=100)

t0 = time()
clf.fit(features_train, labels_train)
print "training time:", round(time()-t0, 3), "s"

t0 = time()
pred = clf.predict(features_test)
print "predicting time:", round(time()-t0, 3), "s"
acc = accuracy_score(pred, labels_test)
print acc

try:
    prettyPicture(clf, features_test, labels_test)
except NameError:
    pass


#! /usr/bin/env python
#
# Copyright (C) 2007-2009 Cournapeau David <cournape@gmail.com>
#               2010 Fabian Pedregosa <fabian.pedregosa@inria.fr>
# License: 3-clause BSD

descr = """A set of python modules for machine learning and data mining"""

import sys
import os
import shutil
from distutils.command.clean import clean as Clean
from pkg_resources import parse_version

if sys.version_info[0] < 3:
    import __builtin__ as builtins
else:
    import builtins

# This is a bit (!) hackish: we are setting a global variable so that the main
# sklearn __init__ can detect if it is being loaded by the setup routine, to
# avoid attempting to load components that aren't built yet:
# the numpy distutils extensions that are used by scikit-learn to recursively
# build the compiled extensions in sub-packages is based on the Python import
# machinery.
builtins.__SKLEARN_SETUP__ = True

DISTNAME = 'scikit-learn'
DESCRIPTION = 'A set of python modules for machine learning and data mining'
with open('README.rst') as f:
    LONG_DESCRIPTION = f.read()
MAINTAINER = 'Andreas Mueller'
MAINTAINER_EMAIL = 'amueller@ais.uni-bonn.de'
URL = 'http://scikit-learn.org'
LICENSE = 'new BSD'
DOWNLOAD_URL = 'http://sourceforge.net/projects/scikit-learn/files/'

# We can actually import a restricted version of sklearn that
# does not need the compiled code
import sklearn
VERSION = sklearn.__version__

# Optional setuptools features
# We need to import setuptools early, if we want setuptools features,
# as it monkey-patches the 'setup' function
# For some commands, use setuptools
SETUPTOOLS_COMMANDS = set([
    'develop', 'release', 'bdist_egg', 'bdist_rpm',
    'bdist_wininst', 'install_egg_info', 'build_sphinx',
    'egg_info', 'easy_install', 'upload', 'bdist_wheel',
    '--single-version-externally-managed',
])
if SETUPTOOLS_COMMANDS.intersection(sys.argv):
    import setuptools
    extra_setuptools_args = dict(
        zip_safe=False,  # the package can run out of an .egg file
        include_package_data=True,
    )
else:
    extra_setuptools_args = dict()

# Custom clean command to remove build artifacts

class CleanCommand(Clean):
    description = "Remove build artifacts from the source tree"

    def run(self):
        Clean.run(self)
        if os.path.exists('build'):
            shutil.rmtree('build')
        for dirpath, dirnames, filenames in os.walk('sklearn'):
            for filename in filenames:
                if (filename.endswith('.so') or filename.endswith('.pyd')
                        or filename.endswith('.dll')
                        or filename.endswith('.pyc')):
                    os.unlink(os.path.join(dirpath, filename))
            for dirname in dirnames:
                if dirname == '__pycache__':
                    shutil.rmtree(os.path.join(dirpath, dirname))

cmdclass = {'clean': CleanCommand}

# Optional wheelhouse-uploader features
# To automate release of binary packages for scikit-learn we need a tool
# to download the packages generated by travis and appveyor workers (with
# version number matching the current release) and upload them all at once
# to PyPI at release time.
# The URL of the artifact repositories are configured in the setup.cfg file.

WHEELHOUSE_UPLOADER_COMMANDS = set(['fetch_artifacts', 'upload_all'])
if WHEELHOUSE_UPLOADER_COMMANDS.intersection(sys.argv):
    import wheelhouse_uploader.cmd
    cmdclass.update(vars(wheelhouse_uploader.cmd))

def configuration(parent_package='', top_path=None):
    if os.path.exists('MANIFEST'):
        os.remove('MANIFEST')

    from numpy.distutils.misc_util import Configuration
    config = Configuration(None, parent_package, top_path)

    # Avoid non-useful msg:
    # "Ignoring attempt to set 'name' (from ... "
    config.set_options(ignore_setup_xxx_py=True,
                       assume_default_configuration=True,
                       delegate_options_to_subpackages=True,
                       quiet=True)

    config.add_subpackage('sklearn')

    return config

scipy_min_version = '0.9'
numpy_min_version = '1.6.1'

def get_scipy_status():
    """
    Returns a dictionary containing a boolean specifying whether SciPy
    is up-to-date, along with the version string (empty string if
    not installed).
    """
    scipy_status = {}
    try:
        import scipy
        scipy_version = scipy.__version__
        scipy_status['up_to_date'] = parse_version(
            scipy_version) >= parse_version(scipy_min_version)
        scipy_status['version'] = scipy_version
    except ImportError:
        scipy_status['up_to_date'] = False
        scipy_status['version'] = ""
    return scipy_status

def get_numpy_status():
    """
    Returns a dictionary containing a boolean specifying whether NumPy
    is up-to-date, along with the version string (empty string if
    not installed).
    """
    numpy_status = {}
    try:
        import numpy
        numpy_version = numpy.__version__
        numpy_status['up_to_date'] = parse_version(
            numpy_version) >= parse_version(numpy_min_version)
        numpy_status['version'] = numpy_version
    except ImportError:
        numpy_status['up_to_date'] = False
        numpy_status['version'] = ""
    return numpy_status

def setup_package():
    metadata = dict(name=DISTNAME,
                    maintainer=MAINTAINER,
                    maintainer_email=MAINTAINER_EMAIL,
                    description=DESCRIPTION,
                    license=LICENSE,
                    url=URL,
                    version=VERSION,
                    download_url=DOWNLOAD_URL,
                    long_description=LONG_DESCRIPTION,
                    classifiers=['Intended Audience :: Science/Research',
                                 'Intended Audience :: Developers',
                                 'License :: OSI Approved',
                                 'Programming Language :: C',
                                 'Programming Language :: Python',
                                 'Topic :: Software Development',
                                 'Topic :: Scientific/Engineering',
                                 'Operating System :: Microsoft :: Windows',
                                 'Operating System :: POSIX',
                                 'Operating System :: Unix',
                                 'Operating System :: MacOS',
                                 'Programming Language :: Python :: 2',
                                 'Programming Language :: Python :: 2.6',
                                 'Programming Language :: Python :: 2.7',
                                 'Programming Language :: Python :: 3',
                                 'Programming Language :: Python :: 3.3',
                                 'Programming Language :: Python :: 3.4',
                                 ],
                    cmdclass=cmdclass,
                    **extra_setuptools_args)

    if (len(sys.argv) >= 2
            and ('--help' in sys.argv[1:] or sys.argv[1]
                 in ('--help-commands', 'egg_info', '--version', 'clean'))):

        # For these actions, NumPy is not required.
        #
        # They are required to succeed without Numpy for example when
        # pip is used to install Scikit-learn when Numpy is not yet present in
        # the system.
        try:
            from setuptools import setup
        except ImportError:
            from distutils.core import setup

        metadata['version'] = VERSION
    else:
        numpy_status = get_numpy_status()
        numpy_req_str = "scikit-learn requires NumPy >= {0}.\n".format(
                        numpy_min_version)
        scipy_status = get_scipy_status()
        scipy_req_str = "scikit-learn requires SciPy >= {0}.\n".format(
                        scipy_min_version)

        instructions = ("Installation instructions are available on the "
                        "scikit-learn website: "
                        "http://scikit-learn.org/stable/install.html\n")

        if numpy_status['up_to_date'] is False:
            if numpy_status['version']:
                raise ImportError("Your installation of Numerical Python "
                                  "(NumPy) {0} is out-of-date.\n{1}{2}"
                                  .format(numpy_status['version'],
                                          numpy_req_str, instructions))
            else:
                raise ImportError("Numerical Python (NumPy) is not "
                                  "installed.\n{0}{1}"
                                  .format(numpy_req_str, instructions))
        if scipy_status['up_to_date'] is False:
            if scipy_status['version']:
                raise ImportError("Your installation of Scientific Python "
                                  "(SciPy) {0} is out-of-date.\n{1}{2}"
                                  .format(scipy_status['version'],
                                          scipy_req_str, instructions))
            else:
                raise ImportError("Scientific Python (SciPy) is not "
                                  "installed.\n{0}{1}"
                                  .format(scipy_req_str, instructions))

        from numpy.distutils.core import setup

        metadata['configuration'] = configuration

    setup(**metadata)

if __name__ == "__main__":
    setup_package()