# Opus/UrbanSim urban simulation software.
# Copyright (C) 2010-2011 University of California, Berkeley, 2005-2009 University of Washington
# See opus_core/LICENSE

import os, re, sys, time, traceback
from copy import copy
from opus_gui.results_manager.run.indicator_framework.visualizer.visualizers.abstract_visualization import Visualization

from opus_core.logger import logger
from numpy import array, arange
from numpy import ones, zeros, hstack, vstack
from numpy import trapz, trim_zeros
from pylab import subplot, plot, show
from pylab import xlabel, ylabel, title, text
from pylab import MultipleLocator, FormatStrFormatter
from pylab import savefig, clf, close

class LorenzCurve(Visualization):

    def __init__(self, source_data, dataset_name,
                 attribute = None,
                 years = None, operation = None, name = None, scale = None,
                 storage_location = None):
        Visualizer.__init__(self, source_data, dataset_name, [attribute],
                                   years, operation, name,
                                   storage_location)
        self._values = None
        self._ginicoeff = None

    def is_single_year_indicator_image_type(self):
        return True

    def get_file_extension(self):
        return 'png'

    def get_visualization_shorthand(self):
        return 'lorenzcurve'

    def get_additional_metadata(self):
        return  {}

    def _create_indicator(self, year):
        """Create a Lorenz Curve for the given indicator,
        save it to the cache directory's 'indicators' sub-directory.
        """

        attribute_short = self.get_attribute_alias(attribute = self.attributes[0],
                                                   year = year)

        title = attribute_short + ' ' + str(year)
        if self.run_description is not None:
            title += '\n' + self.run_description

        # Do calculation
        # Make fresh copy with dtype float64 to avoid overflows
        self._values = array(self._get_indicator(year, wrap = False).astype('float64'))
        self._compute_lorenz()

        file_path = self.get_file_path(year = year)
        self._plot(attribute_short, file_path );

        return file_path

    def _compute_lorenz(self ):
        ''' Do the lorenz curve computation and save the result in the corresponding
        class variables
        '''
        self._values.sort()

        #remove 0 values from array
        self._values = trim_zeros(self._values,'f')

        num_values = self._values.size
        F = arange(1, num_values + 1, 1, "float64")/num_values
        L = self._values.cumsum(dtype="float64")/sum(self._values)
        # Add (0, 0) as the first point for completeness (e.g. plotting)
        origin = array([[0], [0]])
        self._values = vstack((F, L))
        self._values = hstack((origin, self._values))
        # This is the simple form of (0.5 - integral) / 0.5
        self._ginicoeff = 1 - 2 * trapz(self._values[1], self._values[0])

    def _plot(self, attribute_name, file_path=None ):
        clf() # Clear existing plot
        a = self._values[0] * 100
        b = self._values[1] * 100
        ax = subplot(111)
        plot(a, a, 'k--', a, b, 'r')
        ax.set_ylim([0,100])
        ax.grid(color='0.5', linestyle=':', linewidth=0.5)
        xlabel('population')
        ylabel(attribute_name)
        title('Lorenz curve')
        font = {'fontname'   : 'Courier',
          'color'      : 'r',
          'fontweight' : 'bold',
          'fontsize'   : 11
        }
        box = { 'pad' : 6,
          'facecolor' : 'w',
          'linewidth' : 1,
          'fill' : True
        }
        text(5, 90, 'Gini coefficient:  %(gini)f' % {'gini' : self._ginicoeff}, font, color='k', bbox=box )
        majorLocator = MultipleLocator(20)
        majorFormatter = FormatStrFormatter('%d %%')
        minorLocator = MultipleLocator(5)
        ax.xaxis.set_major_locator( majorLocator )
        ax.xaxis.set_major_formatter( majorFormatter)
        ax.xaxis.set_minor_locator( minorLocator )
        ax.yaxis.set_major_locator( majorLocator )
        ax.yaxis.set_major_formatter( majorFormatter)
        ax.yaxis.set_minor_locator( minorLocator )

        if file_path:
            savefig(file_path)
            close()
        else:
            show()

import os
from opus_core.tests import opus_unittest
from numpy import allclose
from opus_gui.results_manager.run.indicator_framework.test_classes.abstract_indicator_test import AbstractIndicatorTest

class Tests(AbstractIndicatorTest):

    def skip_test_create_indicator(self):

        indicator_path = os.path.join(self.temp_cache_path, 'indicators')
        self.assert_(not os.path.exists(indicator_path))

        lorenzcurve = LorenzCurve(
                  source_data = self.source_data,
                  attribute = 'opus_core.test.attribute',
                  dataset_name = 'test',
                  years = None
        )

        lorenzcurve.create(False)

        self.assert_(os.path.exists(indicator_path))
        self.assert_(os.path.exists(os.path.join(indicator_path, 'test__lorenzcurve__attribute__1980.png')))

    def skip_test_perfect_equality(self):
        """Perfect equality is when everybody has the same amount of something"""
        lorenzcurve = LorenzCurve(
                  source_data = self.source_data,
                  attribute = 'opus_core.test.attribute',
                  dataset_name = 'test',
                  years = None
        )
        incomes = ones(100)
        lorenzcurve._values = incomes
        lorenzcurve._compute_lorenz()
        wanted_result = vstack((arange(0, 101) / 100., arange(0, 101) / 100.))
        self.assert_(allclose(lorenzcurve._values, wanted_result))

    def skip_test_perfect_inequality(self):
        """Perfect inequality is when one person has all of something"""
        lorenzcurve = LorenzCurve(
                  source_data = self.source_data,
                  attribute = 'opus_core.test.attribute',
                  dataset_name = 'test',
                  years = None
        )
        incomes = zeros(100)
        incomes[0] = 42
        lorenzcurve._values = incomes
        lorenzcurve._compute_lorenz()
        #We strip all the zero values, so the result consists of only two values
        wanted_result = [[0.,1.],[0.,1.]]
        self.assert_(allclose(lorenzcurve._values, wanted_result))

    def skip_test_small_lorenz(self):
        """Test case for less than 100 people"""
        lorenzcurve = LorenzCurve(
                  source_data = self.source_data,
                  attribute = 'opus_core.test.attribute',
                  dataset_name = 'test',
                  years = None
        )
        incomes = array([1, 1, 2, 3, 4, 5])
        lorenzcurve._values = incomes
        lorenzcurve._compute_lorenz()
        wanted_result = array(
            [[ 0, 1/6.,  2/6.,  3/6.,  4/6.,  5/6.,   6/6. ],
             [ 0, 1/16., 2/16., 4/16., 7/16., 11/16., 16/16. ]])
        self.assert_(allclose(lorenzcurve._values, wanted_result))

    def skip_test_small_gini(self):
        """Test case for gini coefficient for the small case"""
        lorenzcurve = LorenzCurve(
                  source_data = self.source_data,
                  attribute = 'opus_core.test.attribute',
                  dataset_name = 'test',
                  years = None
        )
        incomes = array([1, 1, 2, 3, 4, 5])

        lorenzcurve._values = incomes
        lorenzcurve._compute_lorenz()

        self.assertAlmostEqual(lorenzcurve._ginicoeff, 0.3125)

    def skip_test_large_lorenz(self):
        """Test case for more than 100 people"""
        lorenzcurve = LorenzCurve(
                  source_data = self.source_data,
                  attribute = 'opus_core.test.attribute',
                  dataset_name = 'test',
                  years = None
        )
        incomes = array([731, 700, 619, 450, 419, 512, 232, 266, 131, 188,
                         498, 293, 935, 177, 160, 380, 538, 783, 256, 280,
                         731, 362, 870, 970, 674, 211, 524, 207, 513, 461,
                         280, 275, 410, 282, 144, 682, 573, 252, 382, 909,
                         719, 666, 236, 636, 628, 542, 630, 484, 629, 974,
                         747, 509, 281, 725, 377, 565, 495, 840, 391, 191,
                         929, 679, 217, 179, 336, 562, 293, 881, 271, 172,
                         426, 697, 293, 576, 203, 390, 522, 948, 312, 491,
                         531, 959, 646, 495, 306, 631, 722, 322, 876, 586,
                         316, 124, 796, 250, 456, 112, 661, 294, 749, 619,
                         134, 582, 996, 413, 421, 219, 796, 923, 832, 557])
        lorenzcurve._values = incomes
        lorenzcurve._compute_lorenz()

        wanted_result_F = arange(0, 111) / 110.
        wanted_result_L = array([ 0, 0.00202803,  0.00427335,  0.00664542,  0.00907181,  0.01167928,
        0.01457647,  0.01769094,  0.02089595,  0.02413718,  0.02754138,
        0.03099989,  0.0346757 ,  0.03842393,  0.04224459,  0.0461739 ,
        0.05013943,  0.05434035,  0.0586137 ,  0.06314055,  0.06770362,
        0.07233912,  0.07715569,  0.0820628 ,  0.08704234,  0.09211241,
        0.09718249,  0.10227067,  0.10737696,  0.11268243,  0.1179879 ,
        0.12329338,  0.12861696,  0.13415782,  0.13980734,  0.14552928,
        0.15135987,  0.15744396,  0.16399884,  0.17082534,  0.17770615,
        0.18462318,  0.19168508,  0.19876507,  0.20618911,  0.21366748,
        0.22125448,  0.2288777 ,  0.23659146,  0.2447398 ,  0.25299678,
        0.26134429,  0.27010828,  0.27899902,  0.28796219,  0.29692536,
        0.30594285,  0.31515953,  0.32443052,  0.33371962,  0.34317169,
        0.35265998,  0.36227502,  0.3720168 ,  0.38183102,  0.39191685,
        0.40209322,  0.41232391,  0.42269945,  0.43312932,  0.44366784,
        0.45427878,  0.46548727,  0.47669576,  0.48806721,  0.49945678,
        0.51086445,  0.52229023,  0.53380654,  0.54550393,  0.55747293,
        0.56953247,  0.58173686,  0.5940318 ,  0.60638105,  0.61900192,
        0.63167711,  0.64469634,  0.65776989,  0.67089777,  0.68413428,
        0.6973708 ,  0.71089704,  0.72445949,  0.7386376 ,  0.7530511 ,
        0.7674646 ,  0.78252997,  0.79774019,  0.81349364,  0.82935574,
        0.84530837,  0.86176801,  0.87848115,  0.89530294,  0.91223337,
        0.9293992 ,  0.94676421,  0.9643284 ,  0.98196502,  1.        ])

        self.assert_(allclose(lorenzcurve._values, vstack((wanted_result_F, wanted_result_L))))

if __name__ == '__main__':
    try:
        import matplotlib
    except:
        print 'could not import matplotlib'
    else:
        opus_unittest.main()


from collections import defaultdict
from datetime import datetime
import json
import tensorflow as tf
import os, sys
import pandas as pd

#config dic
H = defaultdict(lambda: None)

#All possible config options:
H['optimizer'] = 'MomentumOptimizer'#'RMSPropOptimizer'
H['learning_rate'] = 0.001
H['momentum'] = 0.9 #0.99
H['kernel_num'] = 16 #32

H['dropout_keep_prob'] = 1.0

H['gpu_fraction'] = 0.9

H['num_classes'] = 2
H['model_name'] = 'resnet2D'

H['pretrained_checkpoint_dir'] = '../luna_resnet2D/output_dir/gold_prio3_plane_mil0'#../luna_resnet2D/output_dir/gen8_20z_3rot_stage1_deep
H['output_dir'] = 'output_dir/old_but_gold_plane_mil0_b4_init_luna' #cross_crop_retrain_zrot
H['predictions_dir'] = ''

H['allow_soft_placement'] = True
H['log_device_placement'] = False
H['max_steps'] = 35
H['MOVING_AVERAGE_DECAY'] = 0.9
H['BATCH_NORM_CENTER'] = True
H['BATCH_NORM_SCALE'] = True
H['weights_initializer'] = 'xavier_initializer' #'xavier_initializer', 'xavier_initializer_conv2d', 'truncated_normal_initializer'
H['gpus'] = [1]
H['summary_step'] = 10

# list iterator
# H['train_lst'] = '../data/multiview-2/tr.lst'
# H['val_lst'] = '../data/multiview-2/va.lst'

H['train_lst'] = '../../../datapipeline_final/dsb3_0/interpolate_candidates_res07/tr_patients_80.lst'
H['val_lst'] = '../../../datapipeline_final/dsb3_0/interpolate_candidates_res07/va_patients_20.lst'

#tr_path = '/media/niklas/Data_3/dsb3/datapipeline_gen9/dsb3_0/interpolate_candidates/cv5/cv/tr' + str(run_id) + '.lst'
#va_path = '/media/niklas/Data_3/dsb3/datapipeline_gen9/dsb3_0/interpolate_candidates/cv5/cv/va' + str(run_id) + '.lst'
#H['train_lst'] = tr_path
#H['val_lst'] = va_path

H['candidate_mode'] = False

# crossed axes options - cross is centrally cropped -> layers are stacked in z-dim
H['num_crossed_layers'] = 1
H['crossed_axes'] = [0,1,2]
H['rand_drop_planes']=0
H['plane_mil'] = False
# y and x image_shape must be equal -> z has same shape!!!
# you can crop if the equal z,y and x in image shape are and smaller than in in_image_shape

# images
# in_image_shapes[1:] must be equal to len of crop_before_loading_in_RAM_ZminZmaxYminYmaxXminXmax
H['in_image_shape'] = [5, 64, 64, 64, 2] #256
# not working #H['crop_before_loading_in_RAM_ZminZmaxYminYmaxXminXmax'] = [False,False,False,False,False,False] # Default = False or None
H['image_shape'] = [5, 3*H['num_crossed_layers'], 64, 64, 2]
H['label_shape'] = [1] #256
H['batch_size'] = 8

#iterator settings
H['load_in_ram'] = True
# due to time consuming operation and quality loss only rotation around one axis is processed randomly chosen
H['rand_rot_axes'] = [0]#,1,2] # 0: z, 1: y, 2: x (attention: x and y rotation lasts long)
H['rand_rot'] = True
H['degree_90_rot'] = H['rand_rot']
H['min_rot_angle'] = -10 #degree
H['max_rot_angle'] = 10 #degree
H['rand_mirror_axes'] = [0,1,2] # 0: z, 1: y, 2: x else False
H['rand_cropping_ZminZmaxYminYmaxXminXmax'] = [False,False,False,False,False,False] # crop within given range # default False: full range

H['save_step'] = 10 # saving checkpoint

H['tr_num_examples'] = len(pd.read_csv(H['train_lst'], header=None, sep='\t'))
H['va_num_examples'] = len(pd.read_csv(H['val_lst'], header=None, sep='\t'))


"""
Testing for grid search module (sklearn.grid_search)

"""

from collections import Iterable, Sized
from sklearn.externals.six.moves import cStringIO as StringIO
from sklearn.externals.six.moves import xrange
from itertools import chain, product
import pickle
import sys
import warnings

import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_raise_message
from sklearn.utils.testing import assert_false, assert_true
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_warns
from sklearn.utils.testing import assert_no_warnings
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.mocking import CheckingClassifier, MockDataFrame

from scipy.stats import distributions

from sklearn.externals.six.moves import zip
from sklearn.base import BaseEstimator
from sklearn.datasets import make_classification
from sklearn.datasets import make_blobs
from sklearn.datasets import make_multilabel_classification
from sklearn.grid_search import (GridSearchCV, RandomizedSearchCV,
                                 ParameterGrid, ParameterSampler,
                                 ChangedBehaviorWarning)
from sklearn.svm import LinearSVC, SVC
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import DecisionTreeClassifier
from sklearn.cluster import KMeans, SpectralClustering
from sklearn.metrics import f1_score
from sklearn.metrics import make_scorer
from sklearn.metrics import roc_auc_score
from sklearn.cross_validation import KFold, StratifiedKFold, FitFailedWarning
from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline

# Neither of the following two estimators inherit from BaseEstimator,
# to test hyperparameter search on user-defined classifiers.
class MockClassifier(object):
    """Dummy classifier to test the cross-validation"""
    def __init__(self, foo_param=0):
        self.foo_param = foo_param

    def fit(self, X, Y):
        assert_true(len(X) == len(Y))
        return self

    def predict(self, T):
        return T.shape[0]

    predict_proba = predict
    decision_function = predict
    transform = predict

    def score(self, X=None, Y=None):
        if self.foo_param > 1:
            score = 1.
        else:
            score = 0.
        return score

    def get_params(self, deep=False):
        return {'foo_param': self.foo_param}

    def set_params(self, **params):
        self.foo_param = params['foo_param']
        return self

class LinearSVCNoScore(LinearSVC):
    """An LinearSVC classifier that has no score method."""
    @property
    def score(self):
        raise AttributeError

X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
y = np.array([1, 1, 2, 2])

def test_parameter_grid():
    """Test basic properties of ParameterGrid."""
    params1 = {"foo": [1, 2, 3]}
    grid1 = ParameterGrid(params1)
    assert_true(isinstance(grid1, Iterable))
    assert_true(isinstance(grid1, Sized))
    assert_equal(len(grid1), 3)

    params2 = {"foo": [4, 2],
               "bar": ["ham", "spam", "eggs"]}
    grid2 = ParameterGrid(params2)
    assert_equal(len(grid2), 6)

    # loop to assert we can iterate over the grid multiple times
    for i in xrange(2):
        # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
        points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
        assert_equal(points,
                     set(("bar", x, "foo", y)
                         for x, y in product(params2["bar"], params2["foo"])))

    # Special case: empty grid (useful to get default estimator settings)
    empty = ParameterGrid({})
    assert_equal(len(empty), 1)
    assert_equal(list(empty), [{}])

    has_empty = ParameterGrid([{'C': [1, 10]}, {}])
    assert_equal(len(has_empty), 3)
    assert_equal(list(has_empty), [{'C': 1}, {'C': 10}, {}])

def test_grid_search():
    """Test that the best estimator contains the right value for foo_param"""
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, verbose=3)
    # make sure it selects the smallest parameter in case of ties
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    grid_search.fit(X, y)
    sys.stdout = old_stdout
    assert_equal(grid_search.best_estimator_.foo_param, 2)

    for i, foo_i in enumerate([1, 2, 3]):
        assert_true(grid_search.grid_scores_[i][0]
                    == {'foo_param': foo_i})
    # Smoke test the score etc:
    grid_search.score(X, y)
    grid_search.predict_proba(X)
    grid_search.decision_function(X)
    grid_search.transform(X)

    # Test exception handling on scoring
    grid_search.scoring = 'sklearn'
    assert_raises(ValueError, grid_search.fit, X, y)

@ignore_warnings
def test_grid_search_no_score():
    # Test grid-search on classifier that has no score function.
    clf = LinearSVC(random_state=0)
    X, y = make_blobs(random_state=0, centers=2)
    Cs = [.1, 1, 10]
    clf_no_score = LinearSVCNoScore(random_state=0)
    grid_search = GridSearchCV(clf, {'C': Cs}, scoring='accuracy')
    grid_search.fit(X, y)

    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs},
                                        scoring='accuracy')
    # smoketest grid search
    grid_search_no_score.fit(X, y)

    # check that best params are equal
    assert_equal(grid_search_no_score.best_params_, grid_search.best_params_)
    # check that we can call score and that it gives the correct result
    assert_equal(grid_search.score(X, y), grid_search_no_score.score(X, y))

    # giving no scoring function raises an error
    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs})
    assert_raise_message(TypeError, "no scoring", grid_search_no_score.fit,
                         [[1]])

def test_grid_search_score_method():
    X, y = make_classification(n_samples=100, n_classes=2, flip_y=.2,
                               random_state=0)
    clf = LinearSVC(random_state=0)
    grid = {'C': [.1]}

    search_no_scoring = GridSearchCV(clf, grid, scoring=None).fit(X, y)
    search_accuracy = GridSearchCV(clf, grid, scoring='accuracy').fit(X, y)
    search_no_score_method_auc = GridSearchCV(LinearSVCNoScore(), grid,
                                              scoring='roc_auc').fit(X, y)
    search_auc = GridSearchCV(clf, grid, scoring='roc_auc').fit(X, y)

    # Check warning only occurs in situation where behavior changed:
    # estimator requires score method to compete with scoring parameter
    score_no_scoring = assert_no_warnings(search_no_scoring.score, X, y)
    score_accuracy = assert_warns(ChangedBehaviorWarning,
                                  search_accuracy.score, X, y)
    score_no_score_auc = assert_no_warnings(search_no_score_method_auc.score,
                                            X, y)
    score_auc = assert_warns(ChangedBehaviorWarning,
                             search_auc.score, X, y)
    # ensure the test is sane
    assert_true(score_auc < 1.0)
    assert_true(score_accuracy < 1.0)
    assert_not_equal(score_auc, score_accuracy)

    assert_almost_equal(score_accuracy, score_no_scoring)
    assert_almost_equal(score_auc, score_no_score_auc)

def test_trivial_grid_scores():
    """Test search over a "grid" with only one point.

    Non-regression test: grid_scores_ wouldn't be set by GridSearchCV.
    """
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1]})
    grid_search.fit(X, y)
    assert_true(hasattr(grid_search, "grid_scores_"))

    random_search = RandomizedSearchCV(clf, {'foo_param': [0]})
    random_search.fit(X, y)
    assert_true(hasattr(random_search, "grid_scores_"))

def test_no_refit():
    """Test that grid search can be used for model selection only"""
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, refit=False)
    grid_search.fit(X, y)
    assert_true(hasattr(grid_search, "best_params_"))

def test_grid_search_error():
    """Test that grid search will capture errors on data with different
    length"""
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, X_[:180], y_)

def test_grid_search_iid():
    # test the iid parameter
    # noise-free simple 2d-data
    X, y = make_blobs(centers=[[0, 0], [1, 0], [0, 1], [1, 1]], random_state=0,
                      cluster_std=0.1, shuffle=False, n_samples=80)
    # split dataset into two folds that are not iid
    # first one contains data of all 4 blobs, second only from two.
    mask = np.ones(X.shape[0], dtype=np.bool)
    mask[np.where(y == 1)[0][::2]] = 0
    mask[np.where(y == 2)[0][::2]] = 0
    # this leads to perfect classification on one fold and a score of 1/3 on
    # the other
    svm = SVC(kernel='linear')
    # create "cv" for splits
    cv = [[mask, ~mask], [~mask, mask]]
    # once with iid=True (default)
    grid_search = GridSearchCV(svm, param_grid={'C': [1, 10]}, cv=cv)
    grid_search.fit(X, y)
    first = grid_search.grid_scores_[0]
    assert_equal(first.parameters['C'], 1)
    assert_array_almost_equal(first.cv_validation_scores, [1, 1. / 3.])
    # for first split, 1/4 of dataset is in test, for second 3/4.
    # take weighted average
    assert_almost_equal(first.mean_validation_score,
                        1 * 1. / 4. + 1. / 3. * 3. / 4.)

    # once with iid=False
    grid_search = GridSearchCV(svm, param_grid={'C': [1, 10]}, cv=cv,
                               iid=False)
    grid_search.fit(X, y)
    first = grid_search.grid_scores_[0]
    assert_equal(first.parameters['C'], 1)
    # scores are the same as above
    assert_array_almost_equal(first.cv_validation_scores, [1, 1. / 3.])
    # averaged score is just mean of scores
    assert_almost_equal(first.mean_validation_score,
                        np.mean(first.cv_validation_scores))

def test_grid_search_one_grid_point():
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
    param_dict = {"C": [1.0], "kernel": ["rbf"], "gamma": [0.1]}

    clf = SVC()
    cv = GridSearchCV(clf, param_dict)
    cv.fit(X_, y_)

    clf = SVC(C=1.0, kernel="rbf", gamma=0.1)
    clf.fit(X_, y_)

    assert_array_equal(clf.dual_coef_, cv.best_estimator_.dual_coef_)

def test_grid_search_bad_param_grid():
    param_dict = {"C": 1.0}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)

    param_dict = {"C": []}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)

    param_dict = {"C": np.ones(6).reshape(3, 2)}
    clf = SVC()
    assert_raises(ValueError, GridSearchCV, clf, param_dict)

def test_grid_search_sparse():
    """Test that grid search works with both dense and sparse matrices"""
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(X_[:180], y_[:180])
    y_pred = cv.predict(X_[180:])
    C = cv.best_estimator_.C

    X_ = sp.csr_matrix(X_)
    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(X_[:180].tocoo(), y_[:180])
    y_pred2 = cv.predict(X_[180:])
    C2 = cv.best_estimator_.C

    assert_true(np.mean(y_pred == y_pred2) >= .9)
    assert_equal(C, C2)

def test_grid_search_sparse_scoring():
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
    cv.fit(X_[:180], y_[:180])
    y_pred = cv.predict(X_[180:])
    C = cv.best_estimator_.C

    X_ = sp.csr_matrix(X_)
    clf = LinearSVC()
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
    cv.fit(X_[:180], y_[:180])
    y_pred2 = cv.predict(X_[180:])
    C2 = cv.best_estimator_.C

    assert_array_equal(y_pred, y_pred2)
    assert_equal(C, C2)
    # Smoke test the score
    #np.testing.assert_allclose(f1_score(cv.predict(X_[:180]), y[:180]),
    #                        cv.score(X_[:180], y[:180]))

    # test loss where greater is worse
    def f1_loss(y_true_, y_pred_):
        return -f1_score(y_true_, y_pred_)
    F1Loss = make_scorer(f1_loss, greater_is_better=False)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring=F1Loss)
    cv.fit(X_[:180], y_[:180])
    y_pred3 = cv.predict(X_[180:])
    C3 = cv.best_estimator_.C

    assert_equal(C, C3)
    assert_array_equal(y_pred, y_pred3)

def test_grid_search_precomputed_kernel():
    """Test that grid search works when the input features are given in the
    form of a precomputed kernel matrix """
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)

    # compute the training kernel matrix corresponding to the linear kernel
    K_train = np.dot(X_[:180], X_[:180].T)
    y_train = y_[:180]

    clf = SVC(kernel='precomputed')
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    cv.fit(K_train, y_train)

    assert_true(cv.best_score_ >= 0)

    # compute the test kernel matrix
    K_test = np.dot(X_[180:], X_[:180].T)
    y_test = y_[180:]

    y_pred = cv.predict(K_test)

    assert_true(np.mean(y_pred == y_test) >= 0)

    # test error is raised when the precomputed kernel is not array-like
    # or sparse
    assert_raises(ValueError, cv.fit, K_train.tolist(), y_train)

def test_grid_search_precomputed_kernel_error_nonsquare():
    """Test that grid search returns an error with a non-square precomputed
    training kernel matrix"""
    K_train = np.zeros((10, 20))
    y_train = np.ones((10, ))
    clf = SVC(kernel='precomputed')
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, K_train, y_train)

def test_grid_search_precomputed_kernel_error_kernel_function():
    """Test that grid search returns an error when using a kernel_function"""
    X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
    kernel_function = lambda x1, x2: np.dot(x1, x2.T)
    clf = SVC(kernel=kernel_function)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
    assert_raises(ValueError, cv.fit, X_, y_)

class BrokenClassifier(BaseEstimator):
    """Broken classifier that cannot be fit twice"""

    def __init__(self, parameter=None):
        self.parameter = parameter

    def fit(self, X, y):
        assert_true(not hasattr(self, 'has_been_fit_'))
        self.has_been_fit_ = True

    def predict(self, X):
        return np.zeros(X.shape[0])

def test_refit():
    """Regression test for bug in refitting

    Simulates re-fitting a broken estimator; this used to break with
    sparse SVMs.
    """
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = GridSearchCV(BrokenClassifier(), [{'parameter': [0, 1]}],
                       scoring="precision", refit=True)
    clf.fit(X, y)

def test_X_as_list():
    """Pass X as list in GridSearchCV"""
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = CheckingClassifier(check_X=lambda x: isinstance(x, list))
    cv = KFold(n=len(X), n_folds=3)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
    grid_search.fit(X.tolist(), y).score(X, y)
    assert_true(hasattr(grid_search, "grid_scores_"))

def test_y_as_list():
    """Pass y as list in GridSearchCV"""
    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    clf = CheckingClassifier(check_y=lambda x: isinstance(x, list))
    cv = KFold(n=len(X), n_folds=3)
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
    grid_search.fit(X, y.tolist()).score(X, y)
    assert_true(hasattr(grid_search, "grid_scores_"))

def test_pandas_input():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((DataFrame, Series))
    except ImportError:
        pass

    X = np.arange(100).reshape(10, 10)
    y = np.array([0] * 5 + [1] * 5)

    for InputFeatureType, TargetType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)

        grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]})
        grid_search.fit(X_df, y_ser).score(X_df, y_ser)
        grid_search.predict(X_df)
        assert_true(hasattr(grid_search, "grid_scores_"))

def test_unsupervised_grid_search():
    # test grid-search with unsupervised estimator
    X, y = make_blobs(random_state=0)
    km = KMeans(random_state=0)
    grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]),
                               scoring='adjusted_rand_score')
    grid_search.fit(X, y)
    # ARI can find the right number :)
    assert_equal(grid_search.best_params_["n_clusters"], 3)

    # Now without a score, and without y
    grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]))
    grid_search.fit(X)
    assert_equal(grid_search.best_params_["n_clusters"], 4)

def test_bad_estimator():
    # test grid-search with clustering algorithm which doesn't support
    # "predict"
    sc = SpectralClustering()
    grid_search = GridSearchCV(sc, param_grid=dict(gamma=[.1, 1, 10]),
                               scoring='ari')
    assert_raise_message(TypeError, "'score' or a 'predict'", grid_search.fit,
                         [[1]])

def test_param_sampler():
    # test basic properties of param sampler
    param_distributions = {"kernel": ["rbf", "linear"],
                           "C": distributions.uniform(0, 1)}
    sampler = ParameterSampler(param_distributions=param_distributions,
                               n_iter=10, random_state=0)
    samples = [x for x in sampler]
    assert_equal(len(samples), 10)
    for sample in samples:
        assert_true(sample["kernel"] in ["rbf", "linear"])
        assert_true(0 <= sample["C"] <= 1)

def test_randomized_search_grid_scores():
    # Make a dataset with a lot of noise to get various kind of prediction
    # errors across CV folds and parameter settings
    X, y = make_classification(n_samples=200, n_features=100, n_informative=3,
                               random_state=0)

    # XXX: as of today (scipy 0.12) it's not possible to set the random seed
    # of scipy.stats distributions: the assertions in this test should thus
    # not depend on the randomization
    params = dict(C=distributions.expon(scale=10),
                  gamma=distributions.expon(scale=0.1))
    n_cv_iter = 3
    n_search_iter = 30
    search = RandomizedSearchCV(SVC(), n_iter=n_search_iter, cv=n_cv_iter,
                                param_distributions=params, iid=False)
    search.fit(X, y)
    assert_equal(len(search.grid_scores_), n_search_iter)

    # Check consistency of the structure of each cv_score item
    for cv_score in search.grid_scores_:
        assert_equal(len(cv_score.cv_validation_scores), n_cv_iter)
        # Because we set iid to False, the mean_validation score is the
        # mean of the fold mean scores instead of the aggregate sample-wise
        # mean score
        assert_almost_equal(np.mean(cv_score.cv_validation_scores),
                            cv_score.mean_validation_score)
        assert_equal(list(sorted(cv_score.parameters.keys())),
                     list(sorted(params.keys())))

    # Check the consistency with the best_score_ and best_params_ attributes
    sorted_grid_scores = list(sorted(search.grid_scores_,
                              key=lambda x: x.mean_validation_score))
    best_score = sorted_grid_scores[-1].mean_validation_score
    assert_equal(search.best_score_, best_score)

    tied_best_params = [s.parameters for s in sorted_grid_scores
                        if s.mean_validation_score == best_score]
    assert_true(search.best_params_ in tied_best_params,
                "best_params_={0} is not part of the"
                " tied best models: {1}".format(
                    search.best_params_, tied_best_params))

def test_grid_search_score_consistency():
    # test that correct scores are used
    clf = LinearSVC(random_state=0)
    X, y = make_blobs(random_state=0, centers=2)
    Cs = [.1, 1, 10]
    for score in ['f1', 'roc_auc']:
        grid_search = GridSearchCV(clf, {'C': Cs}, scoring=score)
        grid_search.fit(X, y)
        cv = StratifiedKFold(n_folds=3, y=y)
        for C, scores in zip(Cs, grid_search.grid_scores_):
            clf.set_params(C=C)
            scores = scores[2]  # get the separate runs from grid scores
            i = 0
            for train, test in cv:
                clf.fit(X[train], y[train])
                if score == "f1":
                    correct_score = f1_score(y[test], clf.predict(X[test]))
                elif score == "roc_auc":
                    dec = clf.decision_function(X[test])
                    correct_score = roc_auc_score(y[test], dec)
                assert_almost_equal(correct_score, scores[i])
                i += 1

def test_pickle():
    """Test that a fit search can be pickled"""
    clf = MockClassifier()
    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, refit=True)
    grid_search.fit(X, y)
    pickle.dumps(grid_search)  # smoke test

    random_search = RandomizedSearchCV(clf, {'foo_param': [1, 2, 3]},
                                       refit=True)
    random_search.fit(X, y)
    pickle.dumps(random_search)  # smoke test

def test_grid_search_with_multioutput_data():
    """ Test search with multi-output estimator"""

    X, y = make_multilabel_classification(return_indicator=True,
                                          random_state=0)

    est_parameters = {"max_depth": [1, 2, 3, 4]}
    cv = KFold(y.shape[0], random_state=0)

    estimators = [DecisionTreeRegressor(random_state=0),
                  DecisionTreeClassifier(random_state=0)]

    # Test with grid search cv
    for est in estimators:
        grid_search = GridSearchCV(est, est_parameters, cv=cv)
        grid_search.fit(X, y)
        for parameters, _, cv_validation_scores in grid_search.grid_scores_:
            est.set_params(**parameters)

            for i, (train, test) in enumerate(cv):
                est.fit(X[train], y[train])
                correct_score = est.score(X[test], y[test])
                assert_almost_equal(correct_score,
                                    cv_validation_scores[i])

    # Test with a randomized search
    for est in estimators:
        random_search = RandomizedSearchCV(est, est_parameters, cv=cv)
        random_search.fit(X, y)
        for parameters, _, cv_validation_scores in random_search.grid_scores_:
            est.set_params(**parameters)

            for i, (train, test) in enumerate(cv):
                est.fit(X[train], y[train])
                correct_score = est.score(X[test], y[test])
                assert_almost_equal(correct_score,
                                    cv_validation_scores[i])

    # Test with a randomized search
    for est in estimators:
        random_search = RandomizedSearchCV(est, est_parameters, cv=cv)
        random_search.fit(X, y)
        for parameters, _, cv_validation_scores in random_search.grid_scores_:
            est.set_params(**parameters)

            for i, (train, test) in enumerate(cv):
                est.fit(X[train], y[train])
                correct_score = est.score(X[test], y[test])
                assert_almost_equal(correct_score,
                                    cv_validation_scores[i])

def test_predict_proba_disabled():
    """Test predict_proba when disabled on estimator."""
    X = np.arange(20).reshape(5, -1)
    y = [0, 0, 1, 1, 1]
    clf = SVC(probability=False)
    gs = GridSearchCV(clf, {}, cv=2).fit(X, y)
    assert_false(hasattr(gs, "predict_proba"))

def test_grid_search_allows_nans():
    """ Test GridSearchCV with Imputer """
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)

class FailingClassifier(BaseEstimator):
    """Classifier that raises a ValueError on fit()"""

    FAILING_PARAMETER = 2

    def __init__(self, parameter=None):
        self.parameter = parameter

    def fit(self, X, y=None):
        if self.parameter == FailingClassifier.FAILING_PARAMETER:
            raise ValueError("Failing classifier failed as required")

    def predict(self, X):
        return np.zeros(X.shape[0])

def test_grid_search_failing_classifier():
    """GridSearchCV with on_error != 'raise'

    Ensures that a warning is raised and score reset where appropriate.
    """

    X, y = make_classification(n_samples=20, n_features=10, random_state=0)

    clf = FailingClassifier()

    # refit=False because we only want to check that errors caused by fits
    # to individual folds will be caught and warnings raised instead. If
    # refit was done, then an exception would be raised on refit and not
    # caught by grid_search (expected behavior), and this would cause an
    # error in this test.
    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score=0.0)

    assert_warns(FitFailedWarning, gs.fit, X, y)

    # Ensure that grid scores were set to zero as required for those fits
    # that are expected to fail.
    assert all(np.all(this_point.cv_validation_scores == 0.0)
               for this_point in gs.grid_scores_
               if this_point.parameters['parameter'] ==
               FailingClassifier.FAILING_PARAMETER)

def test_grid_search_failing_classifier_raise():
    """GridSearchCV with on_error == 'raise' raises the error"""

    X, y = make_classification(n_samples=20, n_features=10, random_state=0)

    clf = FailingClassifier()

    # refit=False because we want to test the behaviour of the grid search part
    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
                      refit=False, error_score='raise')

    # FailingClassifier issues a ValueError so this is what we look for.
    assert_raises(ValueError, gs.fit, X, y)


"""Benchmarks of Singular Value Decomposition (Exact and Approximate)

The data is mostly low rank but is a fat infinite tail.
"""
import gc
from time import time
import numpy as np
from collections import defaultdict

import six

from scipy.linalg import svd
from sklearn.utils.extmath import randomized_svd
from sklearn.datasets.samples_generator import make_low_rank_matrix

def compute_bench(samples_range, features_range, n_iter=3, rank=50):

    it = 0

    results = defaultdict(lambda: [])

    max_it = len(samples_range) * len(features_range)
    for n_samples in samples_range:
        for n_features in features_range:
            it += 1
            print('====================')
            print('Iteration %03d of %03d' % (it, max_it))
            print('====================')
            X = make_low_rank_matrix(n_samples, n_features,
                                  effective_rank=rank,
                                  tail_strength=0.2)

            gc.collect()
            print("benchmarking scipy svd: ")
            tstart = time()
            svd(X, full_matrices=False)
            results['scipy svd'].append(time() - tstart)

            gc.collect()
            print("benchmarking scikit-learn randomized_svd: n_iter=0")
            tstart = time()
            randomized_svd(X, rank, n_iter=0)
            results['scikit-learn randomized_svd (n_iter=0)'].append(
                time() - tstart)

            gc.collect()
            print("benchmarking scikit-learn randomized_svd: n_iter=%d "
                  % n_iter)
            tstart = time()
            randomized_svd(X, rank, n_iter=n_iter)
            results['scikit-learn randomized_svd (n_iter=%d)'
                    % n_iter].append(time() - tstart)

    return results

if __name__ == '__main__':
    from mpl_toolkits.mplot3d import axes3d  # register the 3d projection
    import matplotlib.pyplot as plt

    samples_range = np.linspace(2, 1000, 4).astype(np.int)
    features_range = np.linspace(2, 1000, 4).astype(np.int)
    results = compute_bench(samples_range, features_range)

    label = 'scikit-learn singular value decomposition benchmark results'
    fig = plt.figure(label)
    ax = fig.gca(projection='3d')
    for c, (label, timings) in zip('rbg', sorted(six.iteritems(results))):
        X, Y = np.meshgrid(samples_range, features_range)
        Z = np.asarray(timings).reshape(samples_range.shape[0],
                                        features_range.shape[0])
        # plot the actual surface
        ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3,
                        color=c)
        # dummy point plot to stick the legend to since surface plot do not
        # support legends (yet?)
        ax.plot([1], [1], [1], color=c, label=label)

    ax.set_xlabel('n_samples')
    ax.set_ylabel('n_features')
    ax.set_zlabel('Time (s)')
    ax.legend()
    plt.show()


"""
Constant dollar transformations
===============================================================================

Overview
-------------------------------------------------------------------------------

The function ``const2curr`` computes the equivalent generic cashflow in current
dollars from a generic cashflow in constant dollars of the date given by
``base_date``. ``inflation`` is the inflation rate per compounding period.
``curr2const`` computes the inverse transformation.

Functions in this module
-------------------------------------------------------------------------------

"""
import pandas as pd

# cashflows.
from cashflows.timeseries import *
from cashflows.rate import *
from cashflows.common import *

def const2curr(cflo, inflation, base_date=0):
    """Converts a cashflow of constant dollars to current dollars
    of the time `base_date`.

    Args:
        cflo (pandas.Series): Generic cashflow.
        inflation (pandas.Series): Inflation rate per compounding period.
        base_date (int, str): base date.

    Returns:
        A cashflow in current money (pandas.Series)

    **Examples.**

    >>> cflo=cashflow(const_value=[100] * 5, start='2000', freq='A')
    >>> inflation=interest_rate(const_value=[10, 10, 20, 20, 20], start='2000', freq='A')
    >>> const2curr(cflo=cflo, inflation=inflation) # doctest: +NORMALIZE_WHITESPACE
    2000    100.00
    2001    110.00
    2002    132.00
    2003    158.40
    2004    190.08
    Freq: A-DEC, dtype: float64

    >>> const2curr(cflo=cflo, inflation=inflation, base_date=0) # doctest: +NORMALIZE_WHITESPACE
    2000    100.00
    2001    110.00
    2002    132.00
    2003    158.40
    2004    190.08
    Freq: A-DEC, dtype: float64

    >>> const2curr(cflo=cflo, inflation=inflation, base_date='2000') # doctest: +NORMALIZE_WHITESPACE
    2000    100.00
    2001    110.00
    2002    132.00
    2003    158.40
    2004    190.08
    Freq: A-DEC, dtype: float64

    >>> const2curr(cflo=cflo, inflation=inflation, base_date=4) # doctest: +NORMALIZE_WHITESPACE
    2000     52.609428
    2001     57.870370
    2002     69.444444
    2003     83.333333
    2004    100.000000
    Freq: A-DEC, dtype: float64

    >>> const2curr(cflo=cflo, inflation=inflation, base_date='2004') # doctest: +NORMALIZE_WHITESPACE
    2000     52.609428
    2001     57.870370
    2002     69.444444
    2003     83.333333
    2004    100.000000
    Freq: A-DEC, dtype: float64

    """
    if not isinstance(cflo, pd.Series):
        raise TypeError("cflo must be a TimeSeries object")
    if not isinstance(inflation, pd.Series):
        raise TypeError("inflation must be a TimeSeries object")
    verify_period_range([cflo, inflation])
    factor = to_compound_factor(prate=inflation, base_date=base_date)
    result = cflo.copy()
    for time, _ in enumerate(result):
        result[time] *= factor[time]
    return result

def curr2const(cflo, inflation, base_date=0):
    """Converts a cashflow of current dollars to constant dollars of
    the date `base_date`.

    Args:
        cflo (pandas.Series): Generic cashflow.
        inflation_rate (float, pandas.Series): Inflation rate per compounding period.
        base_date (int): base time..

    Returns:
        A cashflow in constant dollars

    >>> cflo = cashflow(const_value=[100] * 5, start='2015', freq='A')
    >>> inflation = interest_rate(const_value=[10, 10, 20, 20, 20], start='2015', freq='A')
    >>> curr2const(cflo=cflo, inflation=inflation) # doctest: +NORMALIZE_WHITESPACE
    2015    100.000000
    2016     90.909091
    2017     75.757576
    2018     63.131313
    2019     52.609428
    Freq: A-DEC, dtype: float64

    >>> curr2const(cflo=cflo, inflation=inflation, base_date=4) # doctest: +NORMALIZE_WHITESPACE
    2015    190.08
    2016    172.80
    2017    144.00
    2018    120.00
    2019    100.00
    Freq: A-DEC, dtype: float64

    >>> curr2const(cflo=cflo, inflation=inflation, base_date='2017') # doctest: +NORMALIZE_WHITESPACE
    2015    132.000000
    2016    120.000000
    2017    100.000000
    2018     83.333333
    2019     69.444444
    Freq: A-DEC, dtype: float64

    """
    if not isinstance(cflo, pd.Series):
        raise TypeError("cflo must be a TimeSeries object")
    if not isinstance(inflation, pd.Series):
        raise TypeError("inflation must be a TimeSeries object")
    verify_period_range([cflo, inflation])
    factor = to_discount_factor(prate=inflation, base_date=base_date)
    result = cflo.copy()
    for time, _ in enumerate(result):
        result[time] *= factor[time]
    return result

if __name__ == "__main__":
    import doctest
    doctest.testmod()


"""
=============================
Species distribution dataset
=============================

This dataset represents the geographic distribution of species.
The dataset is provided by Phillips et. al. (2006).

The two species are:

 - `"Bradypus variegatus"
   <http://www.iucnredlist.org/apps/redlist/details/3038/0>`_ ,
   the Brown-throated Sloth.

 - `"Microryzomys minutus"
   <http://www.iucnredlist.org/apps/redlist/details/13408/0>`_ ,
   also known as the Forest Small Rice Rat, a rodent that lives in Peru,
   Colombia, Ecuador, Peru, and Venezuela.

References:

 * `"Maximum entropy modeling of species geographic distributions"
   <http://www.cs.princeton.edu/~schapire/papers/ecolmod.pdf>`_
   S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling,
   190:231-259, 2006.

Notes:

 * See examples/applications/plot_species_distribution_modeling.py
   for an example of using this dataset
"""

# Authors: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#          Jake Vanderplas <vanderplas@astro.washington.edu>
#
# License: BSD 3 clause

from io import BytesIO
from os import makedirs
from os.path import exists

try:
    # Python 2
    from urllib2 import urlopen
    PY2 = True
except ImportError:
    # Python 3
    from urllib.request import urlopen
    PY2 = False

import numpy as np

from sklearn.datasets.base import get_data_home, Bunch
from sklearn.datasets.base import _pkl_filepath
from sklearn.externals import joblib

DIRECTORY_URL = "http://www.cs.princeton.edu/~schapire/maxent/datasets/"

SAMPLES_URL = DIRECTORY_URL + "samples.zip"
COVERAGES_URL = DIRECTORY_URL + "coverages.zip"

DATA_ARCHIVE_NAME = "species_coverage.pkz"

def _load_coverage(F, header_length=6, dtype=np.int16):
    """Load a coverage file from an open file object.

    This will return a numpy array of the given dtype
    """
    header = [F.readline() for i in range(header_length)]
    make_tuple = lambda t: (t.split()[0], float(t.split()[1]))
    header = dict([make_tuple(line) for line in header])

    M = np.loadtxt(F, dtype=dtype)
    nodata = int(header[b'NODATA_value'])
    if nodata != -9999:
        M[nodata] = -9999
    return M

def _load_csv(F):
    """Load csv file.

    Parameters
    ----------
    F : file object
        CSV file open in byte mode.

    Returns
    -------
    rec : np.ndarray
        record array representing the data
    """
    if PY2:
        # Numpy recarray wants Python 2 str but not unicode
        names = F.readline().strip().split(',')
    else:
        # Numpy recarray wants Python 3 str but not bytes...
        names = F.readline().decode('ascii').strip().split(',')
    rec = np.loadtxt(F, skiprows=0, delimiter=',', dtype='a22,f4,f4')
    rec.dtype.names = names
    return rec

def construct_grids(batch):
    """Construct the map grid from the batch object

    Parameters
    ----------
    batch : Batch object
        The object returned by :func:`fetch_species_distributions`

    Returns
    -------
    (xgrid, ygrid) : 1-D arrays
        The grid corresponding to the values in batch.coverages
    """
    # x,y coordinates for corner cells
    xmin = batch.x_left_lower_corner + batch.grid_size
    xmax = xmin + (batch.Nx * batch.grid_size)
    ymin = batch.y_left_lower_corner + batch.grid_size
    ymax = ymin + (batch.Ny * batch.grid_size)

    # x coordinates of the grid cells
    xgrid = np.arange(xmin, xmax, batch.grid_size)
    # y coordinates of the grid cells
    ygrid = np.arange(ymin, ymax, batch.grid_size)

    return (xgrid, ygrid)

def fetch_species_distributions(data_home=None,
                                download_if_missing=True):
    """Loader for species distribution dataset from Phillips et. al. (2006)

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    download_if_missing: optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    --------
    The data is returned as a Bunch object with the following attributes:

    coverages : array, shape = [14, 1592, 1212]
        These represent the 14 features measured at each point of the map grid.
        The latitude/longitude values for the grid are discussed below.
        Missing data is represented by the value -9999.

    train : record array, shape = (1623,)
        The training points for the data.  Each point has three fields:

        - train['species'] is the species name
        - train['dd long'] is the longitude, in degrees
        - train['dd lat'] is the latitude, in degrees

    test : record array, shape = (619,)
        The test points for the data.  Same format as the training data.

    Nx, Ny : integers
        The number of longitudes (x) and latitudes (y) in the grid

    x_left_lower_corner, y_left_lower_corner : floats
        The (x,y) position of the lower-left corner, in degrees

    grid_size : float
        The spacing between points of the grid, in degrees

    Notes
    ------

    This dataset represents the geographic distribution of species.
    The dataset is provided by Phillips et. al. (2006).

    The two species are:

    - `"Bradypus variegatus"
      <http://www.iucnredlist.org/apps/redlist/details/3038/0>`_ ,
      the Brown-throated Sloth.

    - `"Microryzomys minutus"
      <http://www.iucnredlist.org/apps/redlist/details/13408/0>`_ ,
      also known as the Forest Small Rice Rat, a rodent that lives in Peru,
      Colombia, Ecuador, Peru, and Venezuela.

    References
    ----------

    * `"Maximum entropy modeling of species geographic distributions"
      <http://www.cs.princeton.edu/~schapire/papers/ecolmod.pdf>`_
      S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling,
      190:231-259, 2006.

    Notes
    -----

    * See examples/applications/plot_species_distribution_modeling.py
      for an example of using this dataset with scikit-learn

    """
    data_home = get_data_home(data_home)
    if not exists(data_home):
        makedirs(data_home)

    # Define parameters for the data files.  These should not be changed
    # unless the data model changes.  They will be saved in the npz file
    # with the downloaded data.
    extra_params = dict(x_left_lower_corner=-94.8,
                        Nx=1212,
                        y_left_lower_corner=-56.05,
                        Ny=1592,
                        grid_size=0.05)
    dtype = np.int16

    archive_path = _pkl_filepath(data_home, DATA_ARCHIVE_NAME)

    if not exists(archive_path):
        print('Downloading species data from %s to %s' % (SAMPLES_URL,
                                                          data_home))
        X = np.load(BytesIO(urlopen(SAMPLES_URL).read()))

        for f in X.files:
            fhandle = BytesIO(X[f])
            if 'train' in f:
                train = _load_csv(fhandle)
            if 'test' in f:
                test = _load_csv(fhandle)

        print('Downloading coverage data from %s to %s' % (COVERAGES_URL,
                                                           data_home))

        X = np.load(BytesIO(urlopen(COVERAGES_URL).read()))

        coverages = []
        for f in X.files:
            fhandle = BytesIO(X[f])
            print(' - converting', f)
            coverages.append(_load_coverage(fhandle))
        coverages = np.asarray(coverages, dtype=dtype)

        bunch = Bunch(coverages=coverages,
                      test=test,
                      train=train,
                      **extra_params)
        joblib.dump(bunch, archive_path, compress=9)
    else:
        bunch = joblib.load(archive_path)

    return bunch


from io import StringIO

import numpy as np
import pytest

from pandas import DataFrame, date_range, read_csv
from pandas.util import _test_decorators as td
from pandas.util.testing import assert_frame_equal

from pandas.io.common import is_gcs_url

def test_is_gcs_url():
    assert is_gcs_url("gcs://pandas/somethingelse.com")
    assert is_gcs_url("gs://pandas/somethingelse.com")
    assert not is_gcs_url("s3://pandas/somethingelse.com")

@td.skip_if_no('gcsfs')
def test_read_csv_gcs(monkeypatch):
    df1 = DataFrame({'int': [1, 3], 'float': [2.0, np.nan], 'str': ['t', 's'],
                     'dt': date_range('2018-06-18', periods=2)})

    class MockGCSFileSystem:
        def open(*args):
            return StringIO(df1.to_csv(index=False))

    monkeypatch.setattr('gcsfs.GCSFileSystem', MockGCSFileSystem)
    df2 = read_csv('gs://test/test.csv', parse_dates=['dt'])

    assert_frame_equal(df1, df2)

@td.skip_if_no('gcsfs')
def test_to_csv_gcs(monkeypatch):
    df1 = DataFrame({'int': [1, 3], 'float': [2.0, np.nan], 'str': ['t', 's'],
                     'dt': date_range('2018-06-18', periods=2)})
    s = StringIO()

    class MockGCSFileSystem:
        def open(*args):
            return s

    monkeypatch.setattr('gcsfs.GCSFileSystem', MockGCSFileSystem)
    df1.to_csv('gs://test/test.csv', index=True)
    df2 = read_csv(StringIO(s.getvalue()), parse_dates=['dt'], index_col=0)

    assert_frame_equal(df1, df2)

@td.skip_if_no('gcsfs')
def test_gcs_get_filepath_or_buffer(monkeypatch):
    df1 = DataFrame({'int': [1, 3], 'float': [2.0, np.nan], 'str': ['t', 's'],
                     'dt': date_range('2018-06-18', periods=2)})

    def mock_get_filepath_or_buffer(*args, **kwargs):
        return (StringIO(df1.to_csv(index=False)),
                None, None, False)

    monkeypatch.setattr('pandas.io.gcs.get_filepath_or_buffer',
                        mock_get_filepath_or_buffer)
    df2 = read_csv('gs://test/test.csv', parse_dates=['dt'])

    assert_frame_equal(df1, df2)

@pytest.mark.skipif(td.safe_import('gcsfs'),
                    reason='Only check when gcsfs not installed')
def test_gcs_not_present_exception():
    with pytest.raises(ImportError) as e:
        read_csv('gs://test/test.csv')
        assert 'gcsfs library is required' in str(e.value)


import SimpleITK as sitk
import numpy as np
import csv
import os
import json
from PIL import Image
import matplotlib.pyplot as plt
import SimpleITK as sitk
from cv2 import imread, imwrite

def load_itk_image(filename):
  itkimage = sitk.ReadImage(filename)
  numpyImage = sitk.GetArrayFromImage(itkimage)
  numpyOrigin = np.array(list(reversed(itkimage.GetOrigin())))
  numpySpacing = np.array(list(reversed(itkimage.GetSpacing())))
  return numpyImage, numpyOrigin, numpySpacing

def readCSV(filename):
  lines = []
  with open(filename, "rb") as f:
    csvreader = csv.reader(f)
    for line in csvreader:
      lines.append(line)
  return lines

def voxel_2_world(voxel_coord, itkimage):
  world_coord = list(reversed(
    itkimage.TransformContinuousIndexToPhysicalPoint(list(reversed(voxel_coord)))))
  return world_coord

def voxelCoordToWorld(voxelCoord, origin, spacing):
  stretchedVoxelCoord = voxelCoord * spacing
  worldCoord = stretchedVoxelCoord + origin
  return worldCoord

def worldToVoxelCoord(worldCoord, origin, spacing):
  stretchedVoxelCoord = np.absolute(worldCoord - origin)
  voxelCoord = stretchedVoxelCoord / spacing
  return voxelCoord

def normalizePlanes(npzarray):
  maxHU = 400.
  minHU = -1000.
  npzarray = (npzarray - minHU) / (maxHU - minHU)
  npzarray[npzarray > 1] = 1.
  npzarray[npzarray < 0] = 0.
  return npzarray

def readFileNameMap(map_filename):
  file_map = {}
  with open(map_filename) as map_file:
    file_name_list = json.load(map_file)
  for it in file_name_list:
    file_map[it['ID_name']] = it['long_name']
  return file_map

def parse_image_file(filename):
  cols = filename.split("-")
  subset = cols[0]
  key = cols[1]
  z_axis = int(cols[2])
  return key, subset[:subset.index('/')], z_axis

def filterBoxes(boxes, threshold):
  filtered_boxes = []
  for box in boxes:
    if box[4] >= threshold:
      filtered_boxes.append(box)
  return filtered_boxes

def readResultMap(result_filename, file_map, threshold):
  result_map = {}
  with open(result_filename) as result_file:
    result_list = json.load(result_file)
  for it in result_list:
    filename = it['file']
    key = file_map[filename]
    key = os.path.splitext(key)[0]
    boxes = it['box']
    boxes = filterBoxes(boxes, threshold)
    if not result_map.get(key):
      result_map[key] = []
    cols = filename.split('_')
    index = int(cols[2])
    result_map[key].append((index, boxes))
  for key, val in result_map.iteritems():
    val.sort()
  return result_map

def readImageMap(filename):
  lines = readCSV(filename)
  result = {}
  for line in lines[1:]:
    worldCoord = np.asarray(
      [float(line[3]), float(line[2]), float(line[1])])
    radius = float(line[4]) / 2.0 + 1.0
    if not result.get(line[0]):
      result[line[0]] = []
    result[line[0]].append((worldCoord, radius))
  return result

def trans(boxes, H, confs, thr = -1.0):
  gw = H['grid_width']
  gh = H['grid_height']
  cell_pix_size = H['region_size']
  rnnl = H['rnn_len']
  ncls = H['num_classes']
  boxes = np.reshape(boxes, (-1, gh, gw, rnnl, 4))
  confs = np.reshape(confs, (-1, gh, gw, rnnl, ncls))
  ret = []
  for i in range(rnnl):
    for y in range(gh):
      for x in range(gw):
        if np.max(confs[0, y, x, i, 1:]) > thr:
          box = boxes[0, y, x, i, :]
          abs_cx = int(box[0]) + cell_pix_size/2 + cell_pix_size * x
          abs_cy = int(box[1]) + cell_pix_size/2 + cell_pix_size * y
          w = box[2]
          h = box[3]
          ret.append([abs_cx, abs_cy, w, h, np.max(confs[0, y, x, i, 1: ])])
  return np.array(ret)

def split(meta_root, samples):
  np.random.seed(2012310818)
  l = len(samples)
  idxes = np.random.permutation(np.arange(l))
  train = [dat[i] for i in idxes[0 : int(l * 0.7)]]
  vals = [dat[i] for i in idxes[int(l * 0.7) : ]]

  with open(meta_root + 'train.json', 'w') as g:
    json.dump(train, g)
  with open(meta_root + 'vals.json', 'w') as g:
    json.dump(vals, g)

def writeCSV(filename, lines):
  with open(filename, "wb") as f:
    csvwriter = csv.writer(f)
    csvwriter.writerows(lines)

def tryFloat(value):
  try:
    value = float(value)
  except:
    value = value

  return value

def getColumn(lines, columnid, elementType=''):
  column = []
  for line in lines:
    try:
      value = line[columnid]
    except:
      continue

    if elementType == 'float':
      value = tryFloat(value)

    column.append(value)
  return column

def mkdir(d):
  if not os.path.exists(d):
    os.mkdir(d)


"""
===================================================================
Decision Tree Regression
===================================================================

A 1D regression with decision tree.

The :ref:`decision trees <tree>` is
used to fit a sine curve with addition noisy observation. As a result, it
learns local linear regressions approximating the sine curve.

We can see that if the maximum depth of the tree (controlled by the
`max_depth` parameter) is set too high, the decision trees learn too fine
details of the training data and learn from the noise, i.e. they overfit.
"""
print(__doc__)

# Import the necessary modules and libraries
import numpy as np
from sklearn.tree import DecisionTreeRegressor
import matplotlib.pyplot as plt

# Create a random dataset
rng = np.random.RandomState(1)
X = np.sort(5 * rng.rand(80, 1), axis=0)
y = np.sin(X).ravel()
y[::5] += 3 * (0.5 - rng.rand(16))

# Fit regression model
regr_1 = DecisionTreeRegressor(max_depth=2)
regr_2 = DecisionTreeRegressor(max_depth=5)
regr_1.fit(X, y)
regr_2.fit(X, y)

# Predict
X_test = np.arange(0.0, 5.0, 0.01)[:, np.newaxis]
y_1 = regr_1.predict(X_test)
y_2 = regr_2.predict(X_test)

# Plot the results
plt.figure()
plt.scatter(X, y, c="darkorange", label="data")
plt.plot(X_test, y_1, color="cornflowerblue", label="max_depth=2", linewidth=2)
plt.plot(X_test, y_2, color="yellowgreen", label="max_depth=5", linewidth=2)
plt.xlabel("data")
plt.ylabel("target")
plt.title("Decision Tree Regression")
plt.legend()
plt.show()


"""
The :mod:`sklearn.metrics.scorer` submodule implements a flexible
interface for model selection and evaluation using
arbitrary score functions.

A scorer object is a callable that can be passed to
:class:`sklearn.grid_search.GridSearchCV` or
:func:`sklearn.cross_validation.cross_val_score` as the ``scoring`` parameter,
to specify how a model should be evaluated.

The signature of the call is ``(estimator, X, y)`` where ``estimator``
is the model to be evaluated, ``X`` is the test data and ``y`` is the
ground truth labeling (or ``None`` in the case of unsupervised models).
"""

# Authors: Andreas Mueller <amueller@ais.uni-bonn.de>
#          Lars Buitinck <L.J.Buitinck@uva.nl>
#          Arnaud Joly <arnaud.v.joly@gmail.com>
# License: Simplified BSD

from abc import ABCMeta, abstractmethod
from functools import partial

import numpy as np

from . import (r2_score, median_absolute_error, mean_absolute_error,
               mean_squared_error, accuracy_score, f1_score,
               roc_auc_score, average_precision_score,
               precision_score, recall_score, log_loss)
from .cluster import adjusted_rand_score
from ..utils.multiclass import type_of_target
from ..externals import six
from ..base import is_regressor

class _BaseScorer(six.with_metaclass(ABCMeta, object)):
    def __init__(self, score_func, sign, kwargs):
        self._kwargs = kwargs
        self._score_func = score_func
        self._sign = sign

    @abstractmethod
    def __call__(self, estimator, X, y, sample_weight=None):
        pass

    def __repr__(self):
        kwargs_string = "".join([", %s=%s" % (str(k), str(v))
                                 for k, v in self._kwargs.items()])
        return ("make_scorer(%s%s%s%s)"
                % (self._score_func.__name__,
                   "" if self._sign > 0 else ", greater_is_better=False",
                   self._factory_args(), kwargs_string))

    def _factory_args(self):
        """Return non-default make_scorer arguments for repr."""
        return ""

class _PredictScorer(_BaseScorer):
    def __call__(self, estimator, X, y_true, sample_weight=None):
        """Evaluate predicted target values for X relative to y_true.

        Parameters
        ----------
        estimator : object
            Trained estimator to use for scoring. Must have a predict_proba
            method; the output of that is used to compute the score.

        X : array-like or sparse matrix
            Test data that will be fed to estimator.predict.

        y_true : array-like
            Gold standard target values for X.

        sample_weight : array-like, optional (default=None)
            Sample weights.

        Returns
        -------
        score : float
            Score function applied to prediction of estimator on X.
        """
        y_pred = estimator.predict(X)
        if sample_weight is not None:
            return self._sign * self._score_func(y_true, y_pred,
                                                 sample_weight=sample_weight,
                                                 **self._kwargs)
        else:
            return self._sign * self._score_func(y_true, y_pred,
                                                 **self._kwargs)

class _ProbaScorer(_BaseScorer):
    def __call__(self, clf, X, y, sample_weight=None):
        """Evaluate predicted probabilities for X relative to y_true.

        Parameters
        ----------
        clf : object
            Trained classifier to use for scoring. Must have a predict_proba
            method; the output of that is used to compute the score.

        X : array-like or sparse matrix
            Test data that will be fed to clf.predict_proba.

        y : array-like
            Gold standard target values for X. These must be class labels,
            not probabilities.

        sample_weight : array-like, optional (default=None)
            Sample weights.

        Returns
        -------
        score : float
            Score function applied to prediction of estimator on X.
        """
        y_pred = clf.predict_proba(X)
        if sample_weight is not None:
            return self._sign * self._score_func(y, y_pred,
                                                 sample_weight=sample_weight,
                                                 **self._kwargs)
        else:
            return self._sign * self._score_func(y, y_pred, **self._kwargs)

    def _factory_args(self):
        return ", needs_proba=True"

class _ThresholdScorer(_BaseScorer):
    def __call__(self, clf, X, y, sample_weight=None):
        """Evaluate decision function output for X relative to y_true.

        Parameters
        ----------
        clf : object
            Trained classifier to use for scoring. Must have either a
            decision_function method or a predict_proba method; the output of
            that is used to compute the score.

        X : array-like or sparse matrix
            Test data that will be fed to clf.decision_function or
            clf.predict_proba.

        y : array-like
            Gold standard target values for X. These must be class labels,
            not decision function values.

        sample_weight : array-like, optional (default=None)
            Sample weights.

        Returns
        -------
        score : float
            Score function applied to prediction of estimator on X.
        """
        y_type = type_of_target(y)
        if y_type not in ("binary", "multilabel-indicator"):
            raise ValueError("{0} format is not supported".format(y_type))

        if is_regressor(clf):
            y_pred = clf.predict(X)
        else:
            try:
                y_pred = clf.decision_function(X)

                # For multi-output multi-class estimator
                if isinstance(y_pred, list):
                    y_pred = np.vstack(p for p in y_pred).T

            except (NotImplementedError, AttributeError):
                y_pred = clf.predict_proba(X)

                if y_type == "binary":
                    y_pred = y_pred[:, 1]
                elif isinstance(y_pred, list):
                    y_pred = np.vstack([p[:, -1] for p in y_pred]).T

        if sample_weight is not None:
            return self._sign * self._score_func(y, y_pred,
                                                 sample_weight=sample_weight,
                                                 **self._kwargs)
        else:
            return self._sign * self._score_func(y, y_pred, **self._kwargs)

    def _factory_args(self):
        return ", needs_threshold=True"

def get_scorer(scoring):
    if isinstance(scoring, six.string_types):
        try:
            scorer = SCORERS[scoring]
        except KeyError:
            raise ValueError('%r is not a valid scoring value. '
                             'Valid options are %s'
                             % (scoring, sorted(SCORERS.keys())))
    else:
        scorer = scoring
    return scorer

def _passthrough_scorer(estimator, *args, **kwargs):
    """Function that wraps estimator.score"""
    return estimator.score(*args, **kwargs)

def check_scoring(estimator, scoring=None, allow_none=False):
    """Determine scorer from user options.

    A TypeError will be thrown if the estimator cannot be scored.

    Parameters
    ----------
    estimator : estimator object implementing 'fit'
        The object to use to fit the data.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    allow_none : boolean, optional, default: False
        If no scoring is specified and the estimator has no score function, we
        can either return None or raise an exception.

    Returns
    -------
    scoring : callable
        A scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
    """
    has_scoring = scoring is not None
    if not hasattr(estimator, 'fit'):
        raise TypeError("estimator should a be an estimator implementing "
                        "'fit' method, %r was passed" % estimator)
    elif has_scoring:
        return get_scorer(scoring)
    elif hasattr(estimator, 'score'):
        return _passthrough_scorer
    elif allow_none:
        return None
    else:
        raise TypeError(
            "If no scoring is specified, the estimator passed should "
            "have a 'score' method. The estimator %r does not." % estimator)

def make_scorer(score_func, greater_is_better=True, needs_proba=False,
                needs_threshold=False, **kwargs):
    """Make a scorer from a performance metric or loss function.

    This factory function wraps scoring functions for use in GridSearchCV
    and cross_val_score. It takes a score function, such as ``accuracy_score``,
    ``mean_squared_error``, ``adjusted_rand_index`` or ``average_precision``
    and returns a callable that scores an estimator's output.

    Read more in the :ref:`User Guide <scoring>`.

    Parameters
    ----------
    score_func : callable,
        Score function (or loss function) with signature
        ``score_func(y, y_pred, **kwargs)``.

    greater_is_better : boolean, default=True
        Whether score_func is a score function (default), meaning high is good,
        or a loss function, meaning low is good. In the latter case, the
        scorer object will sign-flip the outcome of the score_func.

    needs_proba : boolean, default=False
        Whether score_func requires predict_proba to get probability estimates
        out of a classifier.

    needs_threshold : boolean, default=False
        Whether score_func takes a continuous decision certainty.
        This only works for binary classification using estimators that
        have either a decision_function or predict_proba method.

        For example ``average_precision`` or the area under the roc curve
        can not be computed using discrete predictions alone.

    **kwargs : additional arguments
        Additional parameters to be passed to score_func.

    Returns
    -------
    scorer : callable
        Callable object that returns a scalar score; greater is better.

    Examples
    --------
    >>> from sklearn.metrics import fbeta_score, make_scorer
    >>> ftwo_scorer = make_scorer(fbeta_score, beta=2)
    >>> ftwo_scorer
    make_scorer(fbeta_score, beta=2)
    >>> from sklearn.grid_search import GridSearchCV
    >>> from sklearn.svm import LinearSVC
    >>> grid = GridSearchCV(LinearSVC(), param_grid={'C': [1, 10]},
    ...                     scoring=ftwo_scorer)
    """
    sign = 1 if greater_is_better else -1
    if needs_proba and needs_threshold:
        raise ValueError("Set either needs_proba or needs_threshold to True,"
                         " but not both.")
    if needs_proba:
        cls = _ProbaScorer
    elif needs_threshold:
        cls = _ThresholdScorer
    else:
        cls = _PredictScorer
    return cls(score_func, sign, kwargs)

# Standard regression scores
r2_scorer = make_scorer(r2_score)
mean_squared_error_scorer = make_scorer(mean_squared_error,
                                        greater_is_better=False)
mean_absolute_error_scorer = make_scorer(mean_absolute_error,
                                         greater_is_better=False)
median_absolute_error_scorer = make_scorer(median_absolute_error,
                                           greater_is_better=False)

# Standard Classification Scores
accuracy_scorer = make_scorer(accuracy_score)
f1_scorer = make_scorer(f1_score)

# Score functions that need decision values
roc_auc_scorer = make_scorer(roc_auc_score, greater_is_better=True,
                             needs_threshold=True)
average_precision_scorer = make_scorer(average_precision_score,
                                       needs_threshold=True)
precision_scorer = make_scorer(precision_score)
recall_scorer = make_scorer(recall_score)

# Score function for probabilistic classification
log_loss_scorer = make_scorer(log_loss, greater_is_better=False,
                              needs_proba=True)

# Clustering scores
adjusted_rand_scorer = make_scorer(adjusted_rand_score)

SCORERS = dict(r2=r2_scorer,
               median_absolute_error=median_absolute_error_scorer,
               mean_absolute_error=mean_absolute_error_scorer,
               mean_squared_error=mean_squared_error_scorer,
               accuracy=accuracy_scorer, roc_auc=roc_auc_scorer,
               average_precision=average_precision_scorer,
               log_loss=log_loss_scorer,
               adjusted_rand_score=adjusted_rand_scorer)

for name, metric in [('precision', precision_score),
                     ('recall', recall_score), ('f1', f1_score)]:
    SCORERS[name] = make_scorer(metric)
    for average in ['macro', 'micro', 'samples', 'weighted']:
        qualified_name = '{0}_{1}'.format(name, average)
        SCORERS[qualified_name] = make_scorer(partial(metric, pos_label=None,
                                                      average=average))


import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import os
import pickle

def add_scatter(ax,x,y,c='b',m='o',lab='',s=80,drawln=False,alpha=1):
    ax.scatter(x,y, s=s, lw=2.,facecolors='none',edgecolors=c, marker=m,label=lab,alpha=alpha)
    if drawln: ax.plot(x,y, c=c,ls='-')

def draw_unit_sphere(ramin=243.,ramax=246.,dcmin=7.,dcmax=10.,Nran=216000,seed=2015):
    '''# from https://github.com/desihub/imaginglss/master/scripts/imglss-mpi-make-random.py'''
    rng = np.random.RandomState(seed)
    u1,u2= rng.uniform(size=(2, Nran))
    #
    cmin = np.sin(dcmin*np.pi/180)
    cmax = np.sin(dcmax*np.pi/180)
    #
    RA   = ramin + u1*(ramax-ramin)
    DEC  = 90-np.arccos(cmin+u2*(cmax-cmin))*180./np.pi
    return RA,DEC

class QuickRandoms(object):
    '''Draw randomly from unit sphere
    Example:
    qran= QuickRandoms(ramin=243.,ramax=246.,dcmin=7.,dcmax=10.,Nran=216000)
    qran.get_randoms()
    # save and plot
    qran.save_randoms()
    qran.plot(xlim=(244.,244.1),ylim=(8.,8.1)) #,xlim=(244.,244.+10./360),ylim=(8.,8.+10./360.))
    '''
    def __init__(self,ramin=243.,ramax=246.,dcmin=7.,dcmax=10.,Nran=216000):
        self.ramin=ramin
        self.ramax=ramax
        self.dcmin=dcmin
        self.dcmax=dcmax
        self.Nran=Nran

    def get_randoms(self, fn='quick_randoms.pickle'):
        if os.path.exists(fn):
            ra,dec= self.read_randoms()
        else:
            ra,dec=draw_unit_sphere(ramin=self.ramin,ramax=self.ramax,\
                                    dcmin=self.dcmin,dcmax=self.dcmax,Nran=self.Nran)
        self.ra,self.dec=ra,dec

    def save_randoms(self,fn='quick_randoms.pickle'):
        if not os.path.exists(fn):
            fout=open(fn, 'w')
            pickle.dump((self.ra,self.dec),fout)
            fout.close()
            print("Wrote randoms to: %s" % fn)
        else:
            print("WARNING: %s exists, not overwritting it" % fn)

    def read_randoms(self,fn='quick_randoms.pickle'):
        print("Reading randoms from %s" % fn)
        fobj=open(fn, 'r')
        ra,dec= pickle.load(fobj)
        fobj.close()
        return ra,dec

    def plot(self,xlim=None,ylim=None,text=''):
        fig,ax=plt.subplots()
        add_scatter(ax,self.ra,self.dec,c='b',m='.',alpha=0.5)
        ax.set_xlabel('RA')
        ax.set_ylabel('DEC')
        if xlim is not None and ylim is not None:
            ax.set_xlim(xlim)
            ax.set_ylim(ylim)
            text='_xlim%0.5f_%.5f_ylim%.5f_%.5f' % (xlim[0],xlim[1],ylim[0],ylim[1])
        plt.savefig("quick_randoms%s.png" % text)
        plt.close()

class DesiRandoms(object):
    '''Draw randomly from unit sphere & provide 2 masks:
    mask1: inbricks -- indices where ra,dec pts are in LegacySurvey bricks
    mask2: inimages -- union with inbricks and where we have legacy survey imaging data at these ra,dec pts
    Example:
    ran= DesiRandoms(ramin=243.,ramax=246.,dcmin=7.,dcmax=10.,Nran=216000)
    ran.get_randoms()
    # save randoms if file not exist and plot
    ran.save_randoms()
    ran.plot(xlim=(244.,244.1),ylim=(8.,8.1)) #,xlim=(244.,244.+10./360),ylim=(8.,8.+10./360.))
    '''
    def __init__(self,ramin=243.,ramax=246.,dcmin=7.,dcmax=10.,Nran=216000):
        self.ramin=ramin
        self.ramax=ramax
        self.dcmin=dcmin
        self.dcmax=dcmax
        self.Nran=Nran

    def get_randoms(self,fn='desi_randoms.pickle'):
        if os.path.exists(fn):
            self.ra,self.dec,self.i_inbricks,self.i_inimages= self.read_randoms()
        else:
            self.ra,self.dec,self.i_inbricks,self.i_inimages= self.make_randoms()

    def save_randoms(self,fn='desi_randoms.pickle'):
        if not os.path.exists(fn):
            fout=open(fn, 'w')
            pickle.dump((self.ra,self.dec,self.i_inbricks,self.i_inimages),fout)
            fout.close()
            print("Wrote: %s" % fn)
        else:
            print "WARNING: not saving randoms b/c file already exists: %s" % fn

    def make_randoms(self):
        '''Nran -- # of randoms'''
        import imaginglss
        from imaginglss.model import dataproduct
        from imaginglss.model.datarelease import contains
        import h5py
        print "Creating %d Randoms" % self.Nran
        # dr2 cache
        decals = imaginglss.DECALS('/project/projectdirs/desi/users/burleigh/dr3_testdir_for_bb/imaginglss/dr2.conf.py')
        foot= decals.datarelease.create_footprint((self.ramin,self.ramax,self.dcmin,self.dcmax))
        print('Total sq.deg. covered by Bricks= ',foot.area)
        # Sample full ra,dec box
        ra,dec=draw_unit_sphere(ramin=self.ramin,ramax=self.ramax,\
                                dcmin=self.dcmin,dcmax=self.dcmax,Nran=self.Nran)
        #randoms = np.empty(len(ra), dtype=dataproduct.RandomCatalogue)
        #randoms['RA'] = ra
        #randoms['DEC'] = dec
        # Get indices of inbrick points, copied from def filter()
        coord= np.array((ra,dec))
        bid = foot.brickindex.query_internal(coord)
        i_inbricks = contains(foot._covered_brickids, bid)
        i_inbricks = np.where(i_inbricks)[0]
        print('Number Density in bricks= ',len(ra)/foot.area)
        # Union of inbricks and have imaging data, evaluate depths at ra,dec
        coord= coord[:, i_inbricks]
        cat_lim = decals.datarelease.read_depths(coord, 'grz')
        depth= cat_lim['DECAM_DEPTH'] ** -0.5 / cat_lim['DECAM_MW_TRANSMISSION']
        nanmask= np.isnan(depth)
        nanmask=np.all(nanmask[:,[1,2,4]],axis=1) # shape (Nran,)
        i_inimages= i_inbricks[nanmask == False]
        print('ra.size=%d, i_inbricks.size=%d, i_inimages.size=%d' % (ra.size, i_inbricks.size, i_inimages.size))
        # We are not using Yu's randoms dtype=dataproduct.RandomCatalogue
        #randoms['INTRINSIC_NOISELEVEL'][:, :6] = (cat_lim['DECAM_DEPTH'] ** -0.5 / cat_lim['DECAM_MW_TRANSMISSION'])
        #randoms['INTRINSIC_NOISELEVEL'][:, 6:] = 0
        #nanmask = np.isnan(randoms['INTRINSIC_NOISELEVEL'])
        #randoms['INTRINSIC_NOISELEVEL'][nanmask] = np.inf
        print('Total sq.deg. where have imaging data approx.= ',foot.area*(len(ra[i_inimages]))/len(ra))
        print('Number Density for sources where have images= ',len(ra[i_inimages])/foot.area)
        # save ra,dec,mask to file
        #with h5py.File('eboss_randoms.hdf5', 'w') as ff:
        #    ds = ff.create_dataset('_HEADER', shape=(0,))
        #    ds.attrs['FootPrintArea'] = decals.datarelease.footprint.area
        #    ds.attrs['NumberDensity'] = 1.0 * len(randoms) / decals.datarelease.footprint.area
        #    for column in randoms.dtype.names:
        #        ds = ff.create_dataset(column, data=randoms[column])
        #    ds = ff.create_dataset('nanmask', data=nanmask)
        return ra,dec, i_inbricks,i_inimages

    def plot(self,name='desirandoms.png'):
        fig,ax=plt.subplots(1,3,sharey=True,sharex=True,figsize=(15,5))
        add_scatter(ax[0],self.ra, self.dec, c='b',m='o')
        add_scatter(ax[1],self.ra[self.i_inbricks], self.dec[self.i_inbricks], c='b',m='.')
        add_scatter(ax[2],self.ra[self.i_inimages], self.dec[self.i_inimages], c='b',m='.')
        for i,title in zip(range(3),['All','in Bricks','in Images']):
            ti=ax[i].set_title(title)
            xlab=ax[i].set_xlabel('ra')
            ax[i].set_ylim((self.dec.min(),self.dec.max()))
            ax[i].set_xlim((self.ra.min(),self.ra.max()))
        ylab=ax[0].set_ylabel('dec')
        plt.savefig(name, bbox_extra_artists=[ti,xlab,ylab], bbox_inches='tight',dpi=150)
        plt.close()
        print "wrote: %s" % name

class Angular_Correlator(object):
    '''Compute w(theta) from observed ra,dec and random ra,dec
    uses landy szalay estimator: DD - 2DR + RR / RR
    two numerical methods: 1) Yu Feng's kdcount, 2) astroML
    Example:
    ac= Angular_Correlator(gal_ra,gal_dec,ran_ra,ran_dec)
    ac.compute()
    ac.plot()
    '''
    def __init__(self,gal_ra,gal_dec,ran_ra,ran_dec,ncores=1):
        self.gal_ra=gal_ra
        self.gal_dec=gal_dec
        self.ran_ra=ran_ra
        self.ran_dec=ran_dec
        self.ncores=ncores

    def compute(self):
        self.theta,self.w={},{}
        for key in ['astroML','yu']:
            self.theta[key],self.w[key]= self.get_angular_corr(whos=key)
        self.plot()

    def get_angular_corr(self,whos='yu'):
        if whos == 'yu': return self.ac_yu()
        elif whos == 'astroML': return self.ac_astroML()
        else: raise ValueError()

    def ac_astroML(self):
        '''from two_point_angular() in astroML/correlation.py'''
        from astroML.correlation import two_point,ra_dec_to_xyz,angular_dist_to_euclidean_dist
        # 3d project
        data = np.asarray(ra_dec_to_xyz(self.gal_ra, self.gal_dec), order='F').T
        data_R = np.asarray(ra_dec_to_xyz(self.ran_ra, self.ran_dec), order='F').T
        # convert spherical bins to cartesian bins
        bins = 10 ** np.linspace(np.log10(1. / 60.), np.log10(6), 16)
        bins_transform = angular_dist_to_euclidean_dist(bins)
        w= two_point(data, bins_transform, method='landy-szalay',data_R=data_R)
        bin_centers = 0.5 * (bins[1:] + bins[:-1])
        return bin_centers, w

    def ac_yu(self):
        from kdcount import correlate
        from kdcount import sphere
        abin = sphere.AngularBinning(np.logspace(-4, -2.6, 10))
        D = sphere.points(self.gal_ra, self.gal_dec)
        R = sphere.points(self.ran_ra, self.ran_dec) #weights=wt_array
        DD = correlate.paircount(D, D, abin, np=self.ncores)
        DR = correlate.paircount(D, R, abin, np=self.ncores)
        RR = correlate.paircount(R, R, abin, np=self.ncores)
        r = D.norm / R.norm
        w= (DD.sum1 - 2 * r * DR.sum1 + r ** 2 * RR.sum1) / (r ** 2 * RR.sum1)
        return abin.angular_centers,w

    def plot(self,name='wtheta.png'):
        fig,ax=plt.subplots()
        for key,col,mark in zip(['yu','astroML'],['g','b'],['o']*2):
            print "%s: theta,w" % key,self.theta[key],self.w[key]
            add_scatter(ax,self.theta[key], self.w[key], c=col,m=mark,lab=key,alpha=0.5)
        t = np.array([0.01, 10])
        plt.plot(t, 10 * (t / 0.01) ** -0.8, ':k', lw=1)
        ax.legend(loc='upper right',scatterpoints=1)
        xlab=ax.set_xlabel(r'$\theta$ (deg)')
        ylab=ax.set_ylabel(r'$\hat{w}(\theta)$')
        ax.set_xscale('log')
        ax.set_yscale('log')
        plt.savefig(name, bbox_extra_artists=[xlab,ylab], bbox_inches='tight',dpi=150)
        plt.close()
        print "wrote: %s" % name

def ac_unit_test():
    '''angular correlation func unit test'''
    qran= QuickRandoms(ramin=243.,ramax=246.,dcmin=7.,dcmax=10.,Nran=216000)
    qran.get_randoms()
    # subset
    index= np.all((qran.ra >= 244.,qran.ra <= 244.5,\
                   qran.dec >= 8.,qran.dec <= 8.5),axis=0)
    ra,dec= qran.ra[index],qran.dec[index]
    # use these as Ducks for DesiRandoms
    ran= DesiRandoms()
    ran.ra,ran.dec= ra,dec
    index= np.all((ran.ra >= 244.,ran.ra <= 244.25),axis=0)
    ran.i_inbricks= np.where(index)[0]
    index= np.all((index,ran.dec >= 8.1,ran.dec <= 8.4),axis=0)
    ran.i_inimages= np.where(index)[0]
    ran.plot()
    # wtheta
    ac= Angular_Correlator(ran.ra[ran.i_inimages],ran.dec[ran.i_inimages],ran.ra,ran.dec)
    ac.compute()
    ac.plot()
    print 'finished unit_test'

if __name__ == '__main__':
    #ac_unit_test()
    #Nran=int(2.4e3*9.)
    #ran= DesiRandoms(ramin=243.,ramax=246.,dcmin=7.,dcmax=10.,Nran=216000)
    Nran=int(2.4e3*1.e2)
    ran= DesiRandoms(ramin=120.,ramax=130.,dcmin=20.,dcmax=30.,Nran=Nran)
    ran.get_randoms()
    # save randoms if file not exist and plot
    ran.save_randoms(fn='desi_randoms_qual.pickle')
    ran.plot()