#!usr/bin/python

# Copyright 2016, Rohan Dandage <rraadd_8@hotmail.com,rohan@igib.in>
# This program is distributed under General Public License v. 3.

"""
================================
``io_ml``
================================
"""
from os.path import abspath,dirname,exists,basename
from os import makedirs
from sklearn.preprocessing import label_binarize

from dms2dfe.lib.io_data_files import read_pkl,to_pkl
from dms2dfe.lib.io_dfs import set_index,denan,denanrows,del_Unnamed

import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg') # no Xwindows

import matplotlib.pyplot as plt

import warnings
warnings.simplefilter(action = "ignore", category = FutureWarning)
from dms2dfe.lib.io_strs import get_logger
logging=get_logger()
# logging.basicConfig(format='[%(asctime)s] %(levelname)s\tfrom %(filename)s in %(funcName)s(..):%(lineno)d: %(message)s',level=logging.DEBUG) # filename=cfg_xls_fh+'.log'

def corrplot(info):
    """
    Plots a correlation matrix heatmap between range of features and fold change values

    :param info: dict, with the information of the experiment
    """

    from dms2dfe.lib.io_dfs import fhs2data_combo
    from glob import glob
    from dms2dfe.lib.plot_mut_data_heatmaps import clustermap
    from dms2dfe.lib.io_ml_data import make_dXy

    ml_input=info.ml_input
    prj_dh=info.prj_dh
    data_fit_fhs=glob('%s/data_fit/aas/*' % prj_dh)
    data_feats_all_fh='%s/data_feats/aas/data_feats_all' % prj_dh
    data_feats_all=pd.read_csv(data_feats_all_fh).set_index('mutids')
    data_fit_all=fhs2data_combo(data_fit_fhs,['%sA' % ml_input],'mutids')
    data_fit_all.columns=[c.split(': ')[0] for c in data_fit_all]

    for c in data_fit_all:
        plot_fh='%s/plots/aas/%s.corr.pdf' % (prj_dh,c)
        if not exists(plot_fh):
            if not exists(dirname(plot_fh)):
                makedirs(dirname(plot_fh))
            dXy=data_feats_all.join(data_fit_all[c])
            dXy,Xcols,ycol=make_dXy(dXy,ycol=c,
                if_rescalecols=False,
                unique_quantile=0.25)
            dXy,Xcols,ycol=feats_sel_corr(dXy,ycol,range_coef=[0.9,0.8])
            g,ax=clustermap(dXy.corr(method='spearman'),
                        highlight_col=c,
                   vlim=[-0.5,0.5],figsize=[10,10],
                       plot_fh=plot_fh,
           )

def run_RF_classi(data_all,X_cols,y_coln,
           test_size=0.34,data_test=None,data_out_fh=None):
    """
    This implements Random Forest classifier.

    :param data_all: dataframe with columns with features(Xs) and classes(y).
    :param X_cols: list of column names with features.
    :param y_coln: column name of column with classes.
    :param plot_fh: path to output plot file.
    :returns grid_search: trained classifier object.
    :returns y_test: classes used for testing classifier.
    :returns y_pred: predicted classes.
    :returns y_score: scores of predicted classes used to plot ROC curve.
    :returns feature_importances: relative importances of features (dataframe).
    """
    from sklearn.ensemble import RandomForestClassifier

    X=data_all.loc[:,list(X_cols)]
    X=X.as_matrix()

    y=data_all.loc[:,y_coln]
    classes=y.unique()
    y=y.as_matrix()
    y = label_binarize(y, classes=classes)
    if len(classes)==2:
        y=np.array([i[0] for i in y])

    if len(classes)>1:
        if test_size!=0:
            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size,
                                                            random_state=88)
        else :
            X_train=X
            y_train=y
            X_test_df=data_test.loc[:,list(X_cols)]
            X_test_df=denan(X_test_df,axis='both',condi='all any')
            X_test=X_test_df.as_matrix()
            y_test=None

        model = RandomForestClassifier(random_state =88)
        param_grid = {"n_estimators": [1000],
                      "max_features": ['sqrt'],#[None,'sqrt','log2'],
                      "min_samples_leaf":[1],#[1,25,50,100],
                      "criterion": ['entropy'],#["gini", "entropy"]
                     }

        grid_search = GridSearchCV(model, param_grid=param_grid,cv=10)
        grid_search.fit(X_train,y_train)

        y_pred=grid_search.predict(X_test)
        if test_size!=0:
            data_preds=None
        else:
            data_preds=X_test_df
            data_preds[y_coln]=binary2classes(y_pred,classes)

    featimps=pd.DataFrame(columns=['Feature','Importance'])
    featimps.loc[:,'Feature']=X_cols#[indices]
    featimps.loc[:,'Importance']=grid_search.best_estimator_.feature_importances_

    data={'RF_classi':grid_search,
          'X_train':X_train,
          'X_test':X_test,
          'y_train':y_train,
          'y_test':y_test,
          'y_score':grid_search.predict_proba(X_test),
          'classes':classes,
          'X_cols':X_cols,
          'y_coln':y_coln,
          'features':X_cols,
          'featimps':featimps,
          'y_pred':y_pred,
         'data_preds':data_preds}
    to_pkl(data,data_out_fh)
    return grid_search,data_preds

def run_RF_regress(data_all,X_cols,y_coln,
                   test_size=0.5,data_test=None,data_out_fh=None):
    """
    This implements Random Forest classifier.

    :param data_all: dataframe with columns with features(Xs) and classes(y).
    :param X_cols: list of column names with features.
    :param y_coln: column name of column with classes.
    :param plot_fh: path to output plot file.
    :returns grid_search: trained classifier object.
    :returns y_test: classes used for testing classifier.
    :returns y_pred: predicted classes.
    :returns y_score: scores of predicted classes used to plot ROC curve.
    :returns feature_importances: relative importances of features (dataframe).
    """
    from sklearn.ensemble import RandomForestRegressor

    X=data_all.loc[:,list(X_cols)]
    X=X.as_matrix()
    y=data_all.loc[:,y_coln]
    y=y.as_matrix()

    if test_size!=0:
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size,
                                                        random_state=88)
    else :
        X_train=X
        y_train=y
        X_test=data_test.loc[:,list(X_cols)].as_matrix()
        y_test=None

    model = RandomForestRegressor(random_state =88)
    param_grid = {"n_estimators": [3000],#[1000,2000,4000],#
                  "max_features": ['sqrt'],#[None,'sqrt','log2'],
                  "min_samples_leaf":  [1],#[1,25,50,100],
                  "criterion": ["mse"],
                  "oob_score": [True],
                 }

    grid_search = GridSearchCV(model, param_grid=param_grid,cv=10)
    grid_search.fit(X_train,y_train)
    y_pred=grid_search.predict(X_test)

    if test_size!=0:
        data_preds=None
        # print grid_search.score(X_test, y_test)
    else:
        data_preds=data_test.loc[:,list(X_cols)]
        data_preds[y_coln]=y_pred

    featimps=pd.DataFrame(columns=['Feature','Importance'])
    featimps.loc[:,'Feature']=X_cols#[indices]
    featimps.loc[:,'Importance']=grid_search.best_estimator_.feature_importances_

    data={'RF_regress':grid_search,
          'X_train':X_train,
          'X_test':X_test,
          'y_train':y_train,
          'y_test':y_test,
          'X_cols':X_cols,
          'y_coln':y_coln,
          'features':X_cols,
          'featimps':featimps,
          'y_pred':y_pred,
         'data_preds':data_preds}
    to_pkl(data,data_out_fh)
    return grid_search,data_preds

def data_combo2ml(data_combo,data_fn,data_dh,plot_dh,
                ycoln,col_idx,
                ml_type='both',
                middle_percentile_skipped=0.1,
                force=False,
                ):
    """
    This runs the submodules to run classifier from fitness data (`data_combo`).

    :param basename(data_fn): in the form <data_combo>/<aas/cds>/<name of file>.
    :param data_feats: dataframe with features.
    :param y_coln: column name of column with classes (ys).
    :param ml_type: classi | both
    """
    data_combo=del_Unnamed(data_combo)
    for dh in [plot_dh,data_dh]:
        if not exists(dh):
            makedirs(dh)
    # plot_cls_fh="%s/plot_ml_cls_%s.pdf" % (plot_dh,data_fn)
    # plot_reg_fh="%s/plot_ml_reg_%s.pdf" % (plot_dh,data_fn)
    data_combo_fh="%s/%s.input_raw" % (data_dh,data_fn)
    data_fh="%s/%s.cls.all" % (data_dh,data_fn)
    data_cls_train_fh="%s/%s.cls.train" % (data_dh,data_fn)
    data_cls_tests_fh="%s/%s.cls.tests" % (data_dh,data_fn)
    data_reg_train_fh="%s/%s.reg.train" % (data_dh,data_fn)
    data_reg_tests_fh="%s/%s.reg.tests" % (data_dh,data_fn)
    pkld_cls_fh='%s/%s.cls.pkl' % (data_dh,data_fn)
    pkld_reg_fh='%s/%s.reg.pkl' % (data_dh,data_fn)
    # pkld_cls_metrics_fh='%s/%s.cls.metrics.pkl' % (data_dh,data_fn)
    pkld_reg_metrics_fh='%s/%s.reg.metrics.pkl' % (data_dh,data_fn)
    feature_importances_cls_fh="%s_%s_.csv" % (pkld_cls_fh,'featimps')

    y_coln_cls=ycoln
    y_coln_reg=ycoln

    if np.sum(~data_combo.loc[:,y_coln_cls].isnull())<50:
        logging.error("skipping %s: need more data: %d<50" %\
                        (data_fn,np.sum(~data_combo.loc[:,ycoln].isnull())))
        return False

    logging.info("processing: %s" % data_fn)
    if ml_type=='cls' or ml_type=='both':
        if not exists(pkld_cls_fh):
            if not exists(data_cls_train_fh):
                data_combo,data_ml,data_cls_train,data_cls_tests=make_cls_input(data_combo,
                                                                                y_coln_cls,
                                                                        middle_percentile_skipped=middle_percentile_skipped)
                data_combo.to_csv(data_combo_fh)
                data_ml.to_csv(data_fh)
                data_cls_train.to_csv(data_cls_train_fh)
                data_cls_tests.to_csv(data_cls_tests_fh)
            else:
                data_cls_train=pd.read_csv(data_cls_train_fh)
                data_cls_tests=pd.read_csv(data_cls_tests_fh)
                data_cls_train  =data_cls_train.set_index(col_idx,drop=True)
                data_cls_tests  =data_cls_tests.set_index(col_idx,drop=True)
            y_coln_cls="classes"
            logging.info("cls: train set = %d" % len(data_cls_train))
            X_cols_cls=data_cls_train.columns.tolist()
            X_cols_cls.remove(y_coln_cls)
            # cls
            pkld_cls,data_preds=run_RF_classi(data_cls_train,X_cols_cls,y_coln_cls,
                    test_size=0.34,data_out_fh=pkld_cls_fh) #
        else:
            logging.info('already exists: %s' % basename(pkld_cls_fh))
        if not exists(feature_importances_cls_fh):
	        get_RF_classi_metrics(pkld_cls_fh,data_dh=data_dh,plot_dh=plot_dh)

    if ml_type=='both':
        if not exists(pkld_reg_fh):
            if not exists('%s.train' % data_fh):
                data_cls_tests=pd.read_csv(data_cls_train_fh)
                data_cls_train=pd.read_csv(data_cls_tests_fh)
                data_cls_tests  =data_cls_tests.set_index(col_idx,drop=True)
                data_cls_train  =data_cls_train.set_index(col_idx,drop=True)
                feature_importances_cls=pd.read_csv(feature_importances_cls_fh)
                data_reg_train,data_reg_tests=make_reg_input(data_combo,data_cls_train,data_cls_tests,
                                            feature_importances_cls,
                                            y_coln_reg,
                                            y_coln_cls="classes",
                                            topNfeats=25)
                data_reg_train.to_csv(data_reg_train_fh)
                data_reg_tests.to_csv(data_reg_tests_fh)
            else:
                data_reg_train=pd.read_csv(data_cls_train_fh)
                data_reg_tests=pd.read_csv(data_cls_tests_fh)
                data_reg_train  =data_reg_train.set_index(col_idx,drop=True)
                data_reg_tests  =data_reg_tests.set_index(col_idx,drop=True)
            logging.info("reg: train set = %d" % len(data_reg_train))
            X_cols_reg=[c for c in data_reg_train.columns.tolist() if c!=y_coln_reg]
            # print data_reg_train.loc[:,X_cols_reg]
            pkld_reg_metrics,data_preds_reg_metrics=\
            run_RF_regress(data_reg_train,X_cols_reg,y_coln_reg,
                            test_size=0.34,data_out_fh=pkld_reg_metrics_fh)
            get_RF_regress_metrics(pkld_reg_metrics_fh,data_dh=data_dh,plot_dh=plot_dh)
        else:
            logging.info('already exists: %s' % basename(pkld_reg_fh))

def data_regress2data_fit(prj_dh,data_fit_key,
                          data_regress_all,col='FCA_norm'):

    """
    Transforms the fold changes estimated from a regression model in the format of data_fit

    :param prj_dh: path to the project dorectory
    :param data_fit_key: path key to data_fit file
    :param data_regress_all: pandas table with regression estimated fold change values
    """
    # from dms2dfe.lib.io_nums import str2num
    from dms2dfe.lib.io_mut_files import rescale_fitnessbysynonymous,class_fit,mutids_converter

    data_fit=pd.read_csv("%s/%s" % (prj_dh,data_fit_key))
    data_fit=data_fit.loc[:,["mutids",col]].set_index("mutids",drop=True)
    data_fit_combo=data_fit.copy()
    data_fit_inferred=data_regress_all.reset_index().loc[:,["mutids",col]].set_index("mutids",drop=True)
    data_mutids_common=denanrows(data_fit.join(data_fit_inferred.loc[:,col],rsuffix='_inferred'))
    data_mutids_common=data_mutids_common.loc[(data_mutids_common.loc[:,data_mutids_common.columns[0]]!=data_mutids_common.loc[:,data_mutids_common.columns[1]]),:]

    for m in data_fit_combo.index.tolist():
        if pd.isnull(data_fit.loc[m,col]):
            if m in data_fit_inferred.index.tolist():
                data_fit_combo.loc[m,'inferred']=True
                data_fit_combo.loc[m,col]=data_fit_inferred.loc[m,col]
        else:
            data_fit_combo.loc[m,'inferred']=False
    for c in ['refi','ref','mut','refrefi']:
        data_fit_combo.loc[:,c]=mutids_converter(data_fit_combo.index.tolist(), c, 'aas')
    if col=='FCA_norm':
        data_fit_combo=rescale_fitnessbysynonymous(data_fit_combo,col_fit=col,col_fit_rescaled="FiA")
    data_fit_combo=class_fit(data_fit_combo)
    data_fit_combo.loc[:,'FiS']=\
    data_fit_combo.loc[(data_fit_combo.loc[:,'ref']==data_fit_combo.loc[:,'mut']),'FiA']
    data_fit_combo=data_fit_combo.sort_values(by="refi",axis=0)
    data_fit_combo.to_csv("%s/%s_inferred" % (prj_dh,data_fit_key))
    return data_fit_combo

#GB
from dms2dfe.lib.io_strs import get_time
from dms2dfe.lib.io_ml_data import feats_inter,keep_cols,feats_sel_corr,make_dXy,feats_inter_sel_corr
# %run ../../progs/dms2dfe/dms2dfe/lib/io_ml.py
# %run ../../progs/dms2dfe/dms2dfe/lib/io_ml_data.py
# %run ../../1_dms_software/progs/dms2dfe/dms2dfe/lib/io_ml_metrics.py

from sklearn.model_selection import cross_val_predict,cross_val_score
from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble.partial_dependence import plot_partial_dependence,partial_dependence
def run_est(est,X,y,params,cv=True):
    """
    Runs an estimator

    :param est: estimator object
    :param X: predictors (X) values
    :param y: target (y) values
    :param params: additional fitting parameters
    """
    if est=='GBR':
        est = GradientBoostingRegressor(random_state=88)
    elif est=='GBC':
        est = GradientBoostingClassifier(random_state=88)
    est.set_params(**params)
    if cv:
        r2s=cross_val_score(est,X,y,cv=10)
        print [r2s,np.mean(r2s)]
    return r2s,est
def est2feats_imp(est,Xcols,Xy=None):
    """
    Get Feature importances from estimator

    :param est: Estimator object
    :param Xcols: list of column names of predictors
    """
    try:
        feat_imp = pd.DataFrame(est.feature_importances_, Xcols)#.sort_values(ascending=False)
    except:
        est.fit(Xy[0],Xy[1])
        feat_imp = pd.DataFrame(est.feature_importances_, Xcols)#.sort_values(ascending=False)
    feat_imp.columns=['Feature importance']
    feat_imp=feat_imp.sort_values(by='Feature importance',ascending=False)
    return feat_imp

def dXy2ml(dXy,ycol,params=None,
            if_gridsearch=False,
            if_partial_dependence=False,
           if_feats_imps=False,
           inter=None,
           use_top=None,
           out_fh=None,
          regORcls='reg',
           force=False,cores=8):
    """
    Wrapper for ml operations

    :param dXy: pandas table with preditors (X) and target (y) values
    :param ycol: column name of the target column
    """
    if out_fh is None:
        out_fh='%s_%s.pkl' % ('dXy2ml',get_time())

    if exists(out_fh) and (not force):
        try:
            dpkl=read_pkl(out_fh)
        except:
            return False
    else:
        dpkl={}

    if not ('dXy_final' in dpkl.keys()) or force:
        dpkl['dXy_input']=dXy
        dpkl['ycol']=ycol
        dXy_input=dXy.copy()
        to_pkl(dpkl,out_fh) #back
        dXy,Xcols,ycol=make_dXy(dXy,ycol=ycol,
            if_rescalecols=True,
            unique_quantile=0.25)
        if len(dXy)<100:
            return False
        dpkl['dXy_preprocessed']=dXy
        to_pkl(dpkl,out_fh) #back

        dXy,Xcols,ycol=feats_sel_corr(dXy,ycol,range_coef=[0.9,0.8,0.7])
        dpkl['dXy_feats_sel_corr']=dXy
        to_pkl(dpkl,out_fh) #back

        dXy,Xcols,ycol=keep_cols(dXy,dXy_input,ycol)
        dpkl['dXy_feats_indi']=dXy
        to_pkl(dpkl,out_fh) #back

        if inter=='pre':
            dXy,Xcols,ycol=feats_inter_sel_corr(dXy,ycol,Xcols,dpkl['dXy_feats_indi'].copy(),
                                                top_cols=[
         'Conservation score (inverse shannon uncertainty): gaps ignored',#'Conservation score (ConSurf)',
         'Distance from active site residue: minimum',
         'Distance from dimer interface',
         'Temperature factor (flexibility)',
         'Residue depth'])

        dpkl['dXy_feats_inter_sel_corr']=dXy
        dpkl['dXy_final']=dXy
    else:
        dXy_input=dpkl['dXy_input']
        dXy=dpkl['dXy_final']
        ycol=dpkl['ycol']

    to_pkl(dpkl,out_fh) #back

    Xcols=[c for c in dXy.columns.tolist() if c!=ycol]
    X=dXy.loc[:,Xcols].as_matrix()
    y=dXy.loc[:,ycol].as_matrix()
    dpkl['X_final']=X
    dpkl['y_final']=y
    if regORcls=='reg':
        est_method='GBR'
    elif regORcls=='cls':
        est_method='GBC'

    if (if_gridsearch) or (params is None):
        if not ('gs_cv' in dpkl.keys()) or force:
            param_grid = {'learning_rate':[0.005,0.001,0.0001],#[0.1,0.01,0.005],# tuned with n estimators
                          'n_estimators':[1500,2000,3000,5000], # tuned with learning rate
                          'min_samples_leaf':[50,125], # lower -> less overfitting
                          'max_features':[None],
                          'max_depth':[6],
                          'min_samples_split':[int(len(dXy)*0.05),int(len(dXy)*0.1),int(len(dXy)*0.25),int(len(dXy)*0.5)], # 0.5 to 1 of samples
                          'subsample':[0.8],
                          }
            if regORcls=='reg':
                param_grid['loss']=['ls', 'lad', 'huber']
                est_method='GBR'
                est = GradientBoostingRegressor(random_state=88)
            elif regORcls=='cls':
                param_grid['loss']=['deviance', 'exponential']
                est_method='GBC'
                est = GradientBoostingClassifier(random_state=88)
            logging.info('running grid search')
            gs_cv = GridSearchCV(est, param_grid, n_jobs=cores,cv=10).fit(X, y)
            print [gs_cv.best_params_,gs_cv.best_score_]
            params=gs_cv.best_params_
            dpkl['gs_cv']=gs_cv
            to_pkl(dpkl,out_fh) #back
            dpkl['params']=params

    if 'params' in dpkl.keys() and not force:
        params= dpkl['params']
    elif params is None:
        dpkl['params']=params

    if not ('est_all_feats_r2s' in dpkl.keys()) or force:
        r2s,est=run_est(est=est_method,X=X,y=y,params=params)
        dpkl['est_all_feats']=est
        dpkl['est_all_feats_r2s']=r2s

    if not ('feat_imp' in dpkl.keys()) or force:
        if if_gridsearch:
            feat_imp=est2feats_imp(dpkl['gs_cv'].best_estimator_,Xcols,Xy=None)
        else:
            feat_imp=est2feats_imp(est,Xcols,Xy=[X,y])
        dpkl['feat_imp']=feat_imp
        to_pkl(dpkl,out_fh) #back

    if if_feats_imps:
        fig=plt.figure(figsize=(5,10))
        ax=plt.subplot(111)
        feat_imp.plot(kind='barh', title='Feature Importances',ax=ax)
        ax.set_ylabel('Feature Importance Score')
        to_pkl(dpkl,out_fh) #back

    if not use_top is None:
        Xcols=dpkl['feat_imp'].head(use_top).index.tolist() #int(len(feat_imp)*0.15)
#         print Xcols[:use_top//5]
        if inter=='top':
            dXy,Xcols,ycol=feats_inter_sel_corr(dXy,ycol,Xcols,dXy_input,top_cols=Xcols[:len(Xcols)//5])
        X=dXy.loc[:,Xcols].as_matrix()
        y=dXy.loc[:,ycol].as_matrix()
        r2s,est=run_est(est=est_method,X=X,y=y,params=params)
        feat_imp=est2feats_imp(est,Xcols,Xy=[X,y])
        dpkl['feat_imp_top_feats']=feat_imp
        dpkl['dXy_top_feats']=dXy
        dpkl['est_top_feats']=est
        dpkl['est_top_feats_r2s']=r2s
        to_pkl(dpkl,out_fh) #back

    if if_partial_dependence:
        feats_indi=[s for s in Xcols if not ((') ' in s) and (' (' in s))]
        features=[Xcols.index(f) for f in feats_indi]
        fig, axs = plot_partial_dependence(est, X, features,
                                           feature_names=Xcols,
                                           n_jobs=cores, grid_resolution=50,
                                          figsize=[10,30])
    to_pkl(dpkl,out_fh) #back
    # return est,dXy,dpkl

from dms2dfe.lib.io_ml_metrics import get_GB_cls_metrics

def data_fit2ml(dX_fh,dy_fh,info,regORcls='cls'):
    """
    Wrapper for overall data_fit to regression modelling

    :param dX_fh: path to the file containing preditor values
    :param dy_fh: path to the file containing target values
    :param info: dict contaning information about the experiment
    """
    dy=pd.read_csv(dy_fh).set_index('mutids')
    dX=pd.read_csv(dX_fh).set_index('mutids')
    out_fh='%s/data_ml/%s.pkl' % (info.prj_dh,basename(dy_fh))
    if regORcls=='reg':
        ycol='FiA'
        dXy=pd.concat([dy.loc[:,ycol],dX],axis=1)
        dXy.index.name='mutids'
        params={'loss': 'ls', 'learning_rate': 0.001, 'min_samples_leaf': 50, 'n_estimators': 5000, 'subsample': 0.8, 'min_samples_split': 38, 'max_features': None, 'max_depth': 6}
    elif regORcls=='cls':
        ycol='class_fit_binary'
        dy.loc[(dy.loc[:,'class_fit']=='enriched'),ycol]=1
        dy.loc[(dy.loc[:,'class_fit']=='neutral'),ycol]=np.nan
        dy.loc[(dy.loc[:,'class_fit']=='depleted'),ycol]=0
        dXy=pd.concat([dy.loc[:,ycol],dX],axis=1)
        dXy.index.name='mutids'
    #     params={'loss': 'deviance', 'learning_rate': 0.0001, 'min_samples_leaf': 50, 'n_estimators': 3000, 'subsample': 0.8, 'min_samples_split': 23, 'max_features': None, 'max_depth': 6}
        params={'loss': 'exponential', 'learning_rate': 0.001, 'min_samples_leaf': 50, 'n_estimators': 1500, 'subsample': 0.8, 'min_samples_split': 23, 'max_features': None, 'max_depth': 6}
    dXy2ml(dXy,ycol,
    #         params=params,
            if_gridsearch=True,
            if_partial_dependence=False,
    #        if_feats_imps=True,
            out_fh=out_fh,
            inter='pre',
            # force=True,
    #         use_top=25,
            regORcls=regORcls,
            cores=int(info.cores))

    # get metrics plots
    get_GB_cls_metrics(data_fh=out_fh,info=info)


"""Nearest Neighbors graph functions"""

# Author: Jake Vanderplas <vanderplas@astro.washington.edu>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam

import warnings

from .base import KNeighborsMixin, RadiusNeighborsMixin
from .unsupervised import NearestNeighbors

def _check_params(X, metric, p, metric_params):
    """Check the validity of the input parameters"""
    params = zip(['metric', 'p', 'metric_params'],
                 [metric, p, metric_params])
    est_params = X.get_params()
    for param_name, func_param in params:
        if func_param != est_params[param_name]:
            raise ValueError(
                "Got %s for %s, while the estimator has %s for "
                "the same parameter." % (
                    func_param, param_name, est_params[param_name]))

def _query_include_self(X, include_self, mode):
    """Return the query based on include_self param"""
    # Done to preserve backward compatibility.
    if include_self is None:
        if mode == "connectivity":
            warnings.warn(
                "The behavior of 'kneighbors_graph' when mode='connectivity' "
                "will change in version 0.18. Presently, the nearest neighbor "
                "of each sample is the sample itself. Beginning in version "
                "0.18, the default behavior will be to exclude each sample "
                "from being its own nearest neighbor. To maintain the current "
                "behavior, set include_self=True.", DeprecationWarning)
            include_self = True
        else:
            include_self = False

    if include_self:
        query = X._fit_X
    else:
        query = None

    return query

def kneighbors_graph(X, n_neighbors, mode='connectivity', metric='minkowski',
                     p=2, metric_params=None, include_self=None):
    """Computes the (weighted) graph of k-Neighbors for points in X

    Read more in the :ref:`User Guide <unsupervised_neighbors>`.

    Parameters
    ----------
    X : array-like or BallTree, shape = [n_samples, n_features]
        Sample data, in the form of a numpy array or a precomputed
        :class:`BallTree`.

    n_neighbors : int
        Number of neighbors for each sample.

    mode : {'connectivity', 'distance'}, optional
        Type of returned matrix: 'connectivity' will return the
        connectivity matrix with ones and zeros, in 'distance' the
        edges are Euclidean distance between points.

    metric : string, default 'minkowski'
        The distance metric used to calculate the k-Neighbors for each sample
        point. The DistanceMetric class gives a list of available metrics.
        The default distance is 'euclidean' ('minkowski' metric with the p
        param equal to 2.)

    include_self: bool, default backward-compatible.
        Whether or not to mark each sample as the first nearest neighbor to
        itself. If `None`, then True is used for mode='connectivity' and False
        for mode='distance' as this will preserve backwards compatibilty. From
        version 0.18, the default value will be False, irrespective of the
        value of `mode`.

    p : int, default 2
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params: dict, optional
        additional keyword arguments for the metric function.

    Returns
    -------
    A : sparse matrix in CSR format, shape = [n_samples, n_samples]
        A[i, j] is assigned the weight of edge that connects i to j.

    Examples
    --------
    >>> X = [[0], [3], [1]]
    >>> from sklearn.neighbors import kneighbors_graph
    >>> A = kneighbors_graph(X, 2)
    >>> A.toarray()
    array([[ 1.,  0.,  1.],
           [ 0.,  1.,  1.],
           [ 1.,  0.,  1.]])

    See also
    --------
    radius_neighbors_graph
    """
    if not isinstance(X, KNeighborsMixin):
        X = NearestNeighbors(n_neighbors, metric=metric, p=p,
                             metric_params=metric_params).fit(X)
    else:
        _check_params(X, metric, p, metric_params)

    query = _query_include_self(X, include_self, mode)
    return X.kneighbors_graph(X=query, n_neighbors=n_neighbors, mode=mode)

def radius_neighbors_graph(X, radius, mode='connectivity', metric='minkowski',
                           p=2, metric_params=None, include_self=None):
    """Computes the (weighted) graph of Neighbors for points in X

    Neighborhoods are restricted the points at a distance lower than
    radius.

    Read more in the :ref:`User Guide <unsupervised_neighbors>`.

    Parameters
    ----------
    X : array-like or BallTree, shape = [n_samples, n_features]
        Sample data, in the form of a numpy array or a precomputed
        :class:`BallTree`.

    radius : float
        Radius of neighborhoods.

    mode : {'connectivity', 'distance'}, optional
        Type of returned matrix: 'connectivity' will return the
        connectivity matrix with ones and zeros, in 'distance' the
        edges are Euclidean distance between points.

    metric : string, default 'minkowski'
        The distance metric used to calculate the neighbors within a
        given radius for each sample point. The DistanceMetric class
        gives a list of available metrics. The default distance is
        'euclidean' ('minkowski' metric with the param equal to 2.)

    include_self: bool, default None
        Whether or not to mark each sample as the first nearest neighbor to
        itself. If `None`, then True is used for mode='connectivity' and False
        for mode='distance' as this will preserve backwards compatibilty. From
        version 0.18, the default value will be False, irrespective of the
        value of `mode`.

    p : int, default 2
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params: dict, optional
        additional keyword arguments for the metric function.

    Returns
    -------
    A : sparse matrix in CSR format, shape = [n_samples, n_samples]
        A[i, j] is assigned the weight of edge that connects i to j.

    Examples
    --------
    >>> X = [[0], [3], [1]]
    >>> from sklearn.neighbors import radius_neighbors_graph
    >>> A = radius_neighbors_graph(X, 1.5)
    >>> A.toarray()
    array([[ 1.,  0.,  1.],
           [ 0.,  1.,  0.],
           [ 1.,  0.,  1.]])

    See also
    --------
    kneighbors_graph
    """
    if not isinstance(X, RadiusNeighborsMixin):
        X = NearestNeighbors(radius=radius, metric=metric, p=p,
                             metric_params=metric_params).fit(X)
    else:
        _check_params(X, metric, p, metric_params)

    query = _query_include_self(X, include_self, mode)
    return X.radius_neighbors_graph(query, radius, mode)


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Pipelining: chaining a PCA and a logistic regression
=========================================================

The PCA does an unsupervised dimensionality reduction, while the logistic
regression does the prediction.

We use a GridSearchCV to set the dimensionality of the PCA

"""
print(__doc__)

# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model, decomposition, datasets
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV

logistic = linear_model.LogisticRegression()

pca = decomposition.PCA()
pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])

digits = datasets.load_digits()
X_digits = digits.data
y_digits = digits.target

###############################################################################
# Plot the PCA spectrum
pca.fit(X_digits)

plt.figure(1, figsize=(4, 3))
plt.clf()
plt.axes([.2, .2, .7, .7])
plt.plot(pca.explained_variance_, linewidth=2)
plt.axis('tight')
plt.xlabel('n_components')
plt.ylabel('explained_variance_')

###############################################################################
# Prediction

n_components = [20, 40, 64]
Cs = np.logspace(-4, 4, 3)

#Parameters of pipelines can be set using ‘__’ separated parameter names:

estimator = GridSearchCV(pipe,
                         dict(pca__n_components=n_components,
                              logistic__C=Cs))
estimator.fit(X_digits, y_digits)

plt.axvline(estimator.best_estimator_.named_steps['pca'].n_components,
            linestyle=':', label='n_components chosen')
plt.legend(prop=dict(size=12))
plt.show()


# -*- coding: utf-8 -*-
#
# connex_ew.py
#
# This file is part of NEST.
#
# Copyright (C) 2004 The NEST Initiative
#
# NEST is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# NEST is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with NEST.  If not, see <http://www.gnu.org/licenses/>.

"""
NEST spatial example
--------------------

Create two populations of iaf_psc_alpha neurons on a 30x30 grid with edge_wrap,
connect with circular mask, flat probability,
visualize.

BCCN Tutorial @ CNS*09
Hans Ekkehard Plesser, UMB
"""

import matplotlib.pyplot as plt
import numpy as np
import nest

nest.ResetKernel()

pos = nest.spatial.grid(shape=[30, 30], extent=[3., 3.], edge_wrap=True)

#######################################################################
# create and connect two populations
a = nest.Create('iaf_psc_alpha', positions=pos)
b = nest.Create('iaf_psc_alpha', positions=pos)

cdict = {'rule': 'pairwise_bernoulli',
         'p': 0.5,
         'mask': {'circular': {'radius': 0.5}}}

nest.Connect(a, b,
             conn_spec=cdict,
             syn_spec={'weight': nest.random.uniform(0.5, 2.)})

plt.clf()

#####################################################################
# plot targets of neurons in different grid locations

# first, clear existing figure, get current figure
plt.clf()
fig = plt.gcf()

# plot targets of two source neurons into same figure, with mask
for src_index in [30 * 15 + 15, 0]:
    # obtain node id for center
    src = a[src_index:src_index + 1]
    nest.PlotTargets(src, b, mask=cdict['mask'], fig=fig)

# beautify
plt.axes().set_xticks(np.arange(-1.5, 1.55, 0.5))
plt.axes().set_yticks(np.arange(-1.5, 1.55, 0.5))
plt.grid(True)
plt.axis([-2.0, 2.0, -2.0, 2.0])
plt.axes().set_aspect('equal', 'box')
plt.title('Connection targets')

plt.show()

# plt.savefig('connex_ew.pdf')


"""
Base IO code for all datasets
"""

# Copyright (c) 2007 David Cournapeau <cournape@gmail.com>
#               2010 Fabian Pedregosa <fabian.pedregosa@inria.fr>
#               2010 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

import os
import csv
import sys
import shutil
from os import environ
from os.path import dirname
from os.path import join
from os.path import exists
from os.path import expanduser
from os.path import isdir
from os.path import splitext
from os import listdir
from os import makedirs

import numpy as np

from ..utils import check_random_state

class Bunch(dict):
    """Container object for datasets

    Dictionary-like object that exposes its keys as attributes.

    >>> b = Bunch(a=1, b=2)
    >>> b['b']
    2
    >>> b.b
    2
    >>> b.a = 3
    >>> b['a']
    3
    >>> b.c = 6
    >>> b['c']
    6

    """

    def __init__(self, **kwargs):
        super(Bunch, self).__init__(kwargs)

    def __setattr__(self, key, value):
        self[key] = value

    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            raise AttributeError(key)

    def __setstate__(self, state):
        # Bunch pickles generated with scikit-learn 0.16.* have an non
        # empty __dict__. This causes a surprising behaviour when
        # loading these pickles scikit-learn 0.17: reading bunch.key
        # uses __dict__ but assigning to bunch.key use __setattr__ and
        # only changes bunch['key']. More details can be found at:
        # https://github.com/scikit-learn/scikit-learn/issues/6196.
        # Overriding __setstate__ to be a noop has the effect of
        # ignoring the pickled __dict__
        pass

def get_data_home(data_home=None):
    """Return the path of the scikit-learn data dir.

    This folder is used by some large dataset loaders to avoid
    downloading the data several times.

    By default the data dir is set to a folder named 'scikit_learn_data'
    in the user home folder.

    Alternatively, it can be set by the 'SCIKIT_LEARN_DATA' environment
    variable or programmatically by giving an explicit folder path. The
    '~' symbol is expanded to the user home folder.

    If the folder does not already exist, it is automatically created.
    """
    if data_home is None:
        data_home = environ.get('SCIKIT_LEARN_DATA',
                                join('~', 'scikit_learn_data'))
    data_home = expanduser(data_home)
    if not exists(data_home):
        makedirs(data_home)
    return data_home

def clear_data_home(data_home=None):
    """Delete all the content of the data home cache."""
    data_home = get_data_home(data_home)
    shutil.rmtree(data_home)

def load_files(container_path, description=None, categories=None,
               load_content=True, shuffle=True, encoding=None,
               decode_error='strict', random_state=0):
    """Load text files with categories as subfolder names.

    Individual samples are assumed to be files stored a two levels folder
    structure such as the following:

        container_folder/
            category_1_folder/
                file_1.txt
                file_2.txt
                ...
                file_42.txt
            category_2_folder/
                file_43.txt
                file_44.txt
                ...

    The folder names are used as supervised signal label names. The
    individual file names are not important.

    This function does not try to extract features into a numpy array or
    scipy sparse matrix. In addition, if load_content is false it
    does not try to load the files in memory.

    To use text files in a scikit-learn classification or clustering
    algorithm, you will need to use the `sklearn.feature_extraction.text`
    module to build a feature extraction transformer that suits your
    problem.

    If you set load_content=True, you should also specify the encoding of
    the text using the 'encoding' parameter. For many modern text files,
    'utf-8' will be the correct encoding. If you leave encoding equal to None,
    then the content will be made of bytes instead of Unicode, and you will
    not be able to use most functions in `sklearn.feature_extraction.text`.

    Similar feature extractors should be built for other kind of unstructured
    data input such as images, audio, video, ...

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    container_path : string or unicode
        Path to the main folder holding one subfolder per category

    description: string or unicode, optional (default=None)
        A paragraph describing the characteristic of the dataset: its source,
        reference, etc.

    categories : A collection of strings or None, optional (default=None)
        If None (default), load all the categories.
        If not None, list of category names to load (other categories ignored).

    load_content : boolean, optional (default=True)
        Whether to load or not the content of the different files. If
        true a 'data' attribute containing the text information is present
        in the data structure returned. If not, a filenames attribute
        gives the path to the files.

    encoding : string or None (default is None)
        If None, do not try to decode the content of the files (e.g. for
        images or other non-text content).
        If not None, encoding to use to decode text files to Unicode if
        load_content is True.

    decode_error: {'strict', 'ignore', 'replace'}, optional
        Instruction on what to do if a byte sequence is given to analyze that
        contains characters not of the given `encoding`. Passed as keyword
        argument 'errors' to bytes.decode.

    shuffle : bool, optional (default=True)
        Whether or not to shuffle the data: might be important for models that
        make the assumption that the samples are independent and identically
        distributed (i.i.d.), such as stochastic gradient descent.

    random_state : int, RandomState instance or None, optional (default=0)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are: either
        data, the raw text data to learn, or 'filenames', the files
        holding it, 'target', the classification labels (integer index),
        'target_names', the meaning of the labels, and 'DESCR', the full
        description of the dataset.
    """
    target = []
    target_names = []
    filenames = []

    folders = [f for f in sorted(listdir(container_path))
               if isdir(join(container_path, f))]

    if categories is not None:
        folders = [f for f in folders if f in categories]

    for label, folder in enumerate(folders):
        target_names.append(folder)
        folder_path = join(container_path, folder)
        documents = [join(folder_path, d)
                     for d in sorted(listdir(folder_path))]
        target.extend(len(documents) * [label])
        filenames.extend(documents)

    # convert to array for fancy indexing
    filenames = np.array(filenames)
    target = np.array(target)

    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(filenames.shape[0])
        random_state.shuffle(indices)
        filenames = filenames[indices]
        target = target[indices]

    if load_content:
        data = []
        for filename in filenames:
            with open(filename, 'rb') as f:
                data.append(f.read())
        if encoding is not None:
            data = [d.decode(encoding, decode_error) for d in data]
        return Bunch(data=data,
                     filenames=filenames,
                     target_names=target_names,
                     target=target,
                     DESCR=description)

    return Bunch(filenames=filenames,
                 target_names=target_names,
                 target=target,
                 DESCR=description)

def load_iris():
    """Load and return the iris dataset (classification).

    The iris dataset is a classic and very easy multi-class classification
    dataset.

    =================   ==============
    Classes                          3
    Samples per class               50
    Samples total                  150
    Dimensionality                   4
    Features            real, positive
    =================   ==============

    Read more in the :ref:`User Guide <datasets>`.

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the classification labels,
        'target_names', the meaning of the labels, 'feature_names', the
        meaning of the features, and 'DESCR', the
        full description of the dataset.

    Examples
    --------
    Let's say you are interested in the samples 10, 25, and 50, and want to
    know their class name.

    >>> from sklearn.datasets import load_iris
    >>> data = load_iris()
    >>> data.target[[10, 25, 50]]
    array([0, 0, 1])
    >>> list(data.target_names)
    ['setosa', 'versicolor', 'virginica']
    """
    module_path = dirname(__file__)
    with open(join(module_path, 'data', 'iris.csv')) as csv_file:
        data_file = csv.reader(csv_file)
        temp = next(data_file)
        n_samples = int(temp[0])
        n_features = int(temp[1])
        target_names = np.array(temp[2:])
        data = np.empty((n_samples, n_features))
        target = np.empty((n_samples,), dtype=np.int)

        for i, ir in enumerate(data_file):
            data[i] = np.asarray(ir[:-1], dtype=np.float64)
            target[i] = np.asarray(ir[-1], dtype=np.int)

    with open(join(module_path, 'descr', 'iris.rst')) as rst_file:
        fdescr = rst_file.read()

    return Bunch(data=data, target=target,
                 target_names=target_names,
                 DESCR=fdescr,
                 feature_names=['sepal length (cm)', 'sepal width (cm)',
                                'petal length (cm)', 'petal width (cm)'])

def load_breast_cancer():
    """Load and return the breast cancer wisconsin dataset (classification).

    The breast cancer dataset is a classic and very easy binary classification
    dataset.

    =================   ==============
    Classes                          2
    Samples per class    212(M),357(B)
    Samples total                  569
    Dimensionality                  30
    Features            real, positive
    =================   ==============

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the classification labels,
        'target_names', the meaning of the labels, 'feature_names', the
        meaning of the features, and 'DESCR', the
        full description of the dataset.

    The copy of UCI ML Breast Cancer Wisconsin (Diagnostic) dataset is
    downloaded from:
    https://goo.gl/U2Uwz2

    Examples
    --------
    Let's say you are interested in the samples 10, 50, and 85, and want to
    know their class name.

    >>> from sklearn.datasets import load_breast_cancer
    >>> data = load_breast_cancer()
    >>> data.target[[10, 50, 85]]
    array([0, 1, 0])
    >>> list(data.target_names)
    ['malignant', 'benign']
    """
    module_path = dirname(__file__)
    with open(join(module_path, 'data', 'breast_cancer.csv')) as csv_file:
        data_file = csv.reader(csv_file)
        first_line = next(data_file)
        n_samples = int(first_line[0])
        n_features = int(first_line[1])
        target_names = np.array(first_line[2:4])
        data = np.empty((n_samples, n_features))
        target = np.empty((n_samples,), dtype=np.int)

        for count, value in enumerate(data_file):
            data[count] = np.asarray(value[:-1], dtype=np.float64)
            target[count] = np.asarray(value[-1], dtype=np.int)

    with open(join(module_path, 'descr', 'breast_cancer.rst')) as rst_file:
        fdescr = rst_file.read()

    feature_names = np.array(['mean radius', 'mean texture',
                              'mean perimeter', 'mean area',
                              'mean smoothness', 'mean compactness',
                              'mean concavity', 'mean concave points',
                              'mean symmetry', 'mean fractal dimension',
                              'radius error', 'texture error',
                              'perimeter error', 'area error',
                              'smoothness error', 'compactness error',
                              'concavity error', 'concave points error',
                              'symmetry error', 'fractal dimension error',
                              'worst radius', 'worst texture',
                              'worst perimeter', 'worst area',
                              'worst smoothness', 'worst compactness',
                              'worst concavity', 'worst concave points',
                              'worst symmetry', 'worst fractal dimension'])

    return Bunch(data=data, target=target,
                 target_names=target_names,
                 DESCR=fdescr,
                 feature_names=feature_names)

def load_digits(n_class=10):
    """Load and return the digits dataset (classification).

    Each datapoint is a 8x8 image of a digit.

    =================   ==============
    Classes                         10
    Samples per class             ~180
    Samples total                 1797
    Dimensionality                  64
    Features             integers 0-16
    =================   ==============

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    n_class : integer, between 0 and 10, optional (default=10)
        The number of classes to return.

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'images', the images corresponding
        to each sample, 'target', the classification labels for each
        sample, 'target_names', the meaning of the labels, and 'DESCR',
        the full description of the dataset.

    Examples
    --------
    To load the data and visualize the images::

        >>> from sklearn.datasets import load_digits
        >>> digits = load_digits()
        >>> print(digits.data.shape)
        (1797, 64)
        >>> import matplotlib.pyplot as plt #doctest: +SKIP
        >>> plt.gray() #doctest: +SKIP
        >>> plt.matshow(digits.images[0]) #doctest: +SKIP
        >>> plt.show() #doctest: +SKIP
    """
    module_path = dirname(__file__)
    data = np.loadtxt(join(module_path, 'data', 'digits.csv.gz'),
                      delimiter=',')
    with open(join(module_path, 'descr', 'digits.rst')) as f:
        descr = f.read()
    target = data[:, -1]
    flat_data = data[:, :-1]
    images = flat_data.view()
    images.shape = (-1, 8, 8)

    if n_class < 10:
        idx = target < n_class
        flat_data, target = flat_data[idx], target[idx]
        images = images[idx]

    return Bunch(data=flat_data,
                 target=target.astype(np.int),
                 target_names=np.arange(10),
                 images=images,
                 DESCR=descr)

def load_diabetes():
    """Load and return the diabetes dataset (regression).

    ==============      ==================
    Samples total       442
    Dimensionality      10
    Features            real, -.2 < x < .2
    Targets             integer 25 - 346
    ==============      ==================

    Read more in the :ref:`User Guide <datasets>`.

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn and 'target', the regression target for each
        sample.
    """
    base_dir = join(dirname(__file__), 'data')
    data = np.loadtxt(join(base_dir, 'diabetes_data.csv.gz'))
    target = np.loadtxt(join(base_dir, 'diabetes_target.csv.gz'))
    return Bunch(data=data, target=target)

def load_linnerud():
    """Load and return the linnerud dataset (multivariate regression).

    Samples total: 20
    Dimensionality: 3 for both data and targets
    Features: integer
    Targets: integer

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are: 'data' and
        'targets', the two multivariate datasets, with 'data' corresponding to
        the exercise and 'targets' corresponding to the physiological
        measurements, as well as 'feature_names' and 'target_names'.
    """
    base_dir = join(dirname(__file__), 'data/')
    # Read data
    data_exercise = np.loadtxt(base_dir + 'linnerud_exercise.csv', skiprows=1)
    data_physiological = np.loadtxt(base_dir + 'linnerud_physiological.csv',
                                    skiprows=1)
    # Read header
    with open(base_dir + 'linnerud_exercise.csv') as f:
        header_exercise = f.readline().split()
    with open(base_dir + 'linnerud_physiological.csv') as f:
        header_physiological = f.readline().split()
    with open(dirname(__file__) + '/descr/linnerud.rst') as f:
        descr = f.read()

    return Bunch(data=data_exercise, feature_names=header_exercise,
                 target=data_physiological,
                 target_names=header_physiological,
                 DESCR=descr)

def load_boston():
    """Load and return the boston house-prices dataset (regression).

    ==============     ==============
    Samples total                 506
    Dimensionality                 13
    Features           real, positive
    Targets             real 5. - 50.
    ==============     ==============

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the regression targets,
        and 'DESCR', the full description of the dataset.

    Examples
    --------
    >>> from sklearn.datasets import load_boston
    >>> boston = load_boston()
    >>> print(boston.data.shape)
    (506, 13)
    """
    module_path = dirname(__file__)

    fdescr_name = join(module_path, 'descr', 'boston_house_prices.rst')
    with open(fdescr_name) as f:
        descr_text = f.read()

    data_file_name = join(module_path, 'data', 'boston_house_prices.csv')
    with open(data_file_name) as f:
        data_file = csv.reader(f)
        temp = next(data_file)
        n_samples = int(temp[0])
        n_features = int(temp[1])
        data = np.empty((n_samples, n_features))
        target = np.empty((n_samples,))
        temp = next(data_file)  # names of features
        feature_names = np.array(temp)

        for i, d in enumerate(data_file):
            data[i] = np.asarray(d[:-1], dtype=np.float64)
            target[i] = np.asarray(d[-1], dtype=np.float64)

    return Bunch(data=data,
                 target=target,
                 # last column is target value
                 feature_names=feature_names[:-1],
                 DESCR=descr_text)

def load_sample_images():
    """Load sample images for image manipulation.
    Loads both, ``china`` and ``flower``.

    Returns
    -------
    data : Bunch
        Dictionary-like object with the following attributes :
        'images', the two sample images, 'filenames', the file
        names for the images, and 'DESCR'
        the full description of the dataset.

    Examples
    --------
    To load the data and visualize the images:

    >>> from sklearn.datasets import load_sample_images
    >>> dataset = load_sample_images()     #doctest: +SKIP
    >>> len(dataset.images)                #doctest: +SKIP
    2
    >>> first_img_data = dataset.images[0] #doctest: +SKIP
    >>> first_img_data.shape               #doctest: +SKIP
    (427, 640, 3)
    >>> first_img_data.dtype               #doctest: +SKIP
    dtype('uint8')
    """
    # Try to import imread from scipy. We do this lazily here to prevent
    # this module from depending on PIL.
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
    except ImportError:
        raise ImportError("The Python Imaging Library (PIL) "
                          "is required to load data from jpeg files")
    module_path = join(dirname(__file__), "images")
    with open(join(module_path, 'README.txt')) as f:
        descr = f.read()
    filenames = [join(module_path, filename)
                 for filename in os.listdir(module_path)
                 if filename.endswith(".jpg")]
    # Load image data for each image in the source folder.
    images = [imread(filename) for filename in filenames]

    return Bunch(images=images,
                 filenames=filenames,
                 DESCR=descr)

def load_sample_image(image_name):
    """Load the numpy array of a single sample image

    Parameters
    -----------
    image_name: {`china.jpg`, `flower.jpg`}
        The name of the sample image loaded

    Returns
    -------
    img: 3D array
        The image as a numpy array: height x width x color

    Examples
    ---------

    >>> from sklearn.datasets import load_sample_image
    >>> china = load_sample_image('china.jpg')   # doctest: +SKIP
    >>> china.dtype                              # doctest: +SKIP
    dtype('uint8')
    >>> china.shape                              # doctest: +SKIP
    (427, 640, 3)
    >>> flower = load_sample_image('flower.jpg') # doctest: +SKIP
    >>> flower.dtype                             # doctest: +SKIP
    dtype('uint8')
    >>> flower.shape                             # doctest: +SKIP
    (427, 640, 3)
    """
    images = load_sample_images()
    index = None
    for i, filename in enumerate(images.filenames):
        if filename.endswith(image_name):
            index = i
            break
    if index is None:
        raise AttributeError("Cannot find sample image: %s" % image_name)
    return images.images[index]

def _pkl_filepath(*args, **kwargs):
    """Ensure different filenames for Python 2 and Python 3 pickles

    An object pickled under Python 3 cannot be loaded under Python 2.
    An object pickled under Python 2 can sometimes not be loaded loaded
    correctly under Python 3 because some Python 2 strings are decoded as
    Python 3 strings which can be problematic for objects that use Python 2
    strings as byte buffers for numerical data instead of "real" strings.

    Therefore, dataset loaders in scikit-learn use different files for pickles
    manages by Python 2 and Python 3 in the same SCIKIT_LEARN_DATA folder so
    as to avoid conflicts.

    args[-1] is expected to be the ".pkl" filename. Under Python 3, a
    suffix is inserted before the extension to s

    _pkl_filepath('/path/to/folder', 'filename.pkl') returns:
      - /path/to/folder/filename.pkl under Python 2
      - /path/to/folder/filename_py3.pkl under Python 3+

    """
    py3_suffix = kwargs.get("py3_suffix", "_py3")
    basename, ext = splitext(args[-1])
    if sys.version_info[0] >= 3:
        basename += py3_suffix
    new_args = args[:-1] + (basename + ext,)
    return join(*new_args)


import numpy as np
from numpy.testing import assert_array_almost_equal

from nose.tools import assert_raises
from sklearn.manifold import mds

def test_smacof():
    # test metric smacof using the data of "Modern Multidimensional Scaling",
    # Borg & Groenen, p 154
    sim = np.array([[0, 5, 3, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])
    Z = np.array([[-.266, -.539],
                  [.451, .252],
                  [.016, -.238],
                  [-.200, .524]])
    X, _ = mds.smacof(sim, init=Z, n_components=2, max_iter=1, n_init=1)
    X_true = np.array([[-1.415, -2.471],
                       [1.633, 1.107],
                       [.249, -.067],
                       [-.468, 1.431]])
    assert_array_almost_equal(X, X_true, decimal=3)

def test_smacof_error():
    # Not symmetric similarity matrix:
    sim = np.array([[0, 5, 9, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])

    assert_raises(ValueError, mds.smacof, sim)

    # Not squared similarity matrix:
    sim = np.array([[0, 5, 9, 4],
                    [5, 0, 2, 2],
                    [4, 2, 1, 0]])

    assert_raises(ValueError, mds.smacof, sim)

    # init not None and not correct format:
    sim = np.array([[0, 5, 3, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])

    Z = np.array([[-.266, -.539],
                  [.016, -.238],
                  [-.200, .524]])
    assert_raises(ValueError, mds.smacof, sim, init=Z, n_init=1)

def test_MDS():
    sim = np.array([[0, 5, 3, 4],
                    [5, 0, 2, 2],
                    [3, 2, 0, 1],
                    [4, 2, 1, 0]])
    mds_clf = mds.MDS(metric=False, n_jobs=3, dissimilarity="precomputed")
    mds_clf.fit(sim)


import colorsys
import numpy as np
import matplotlib as mpl

import pytest
import nose.tools as nt
import numpy.testing as npt
import matplotlib.pyplot as plt

from .. import palettes, utils, rcmod
from ..external import husl
from ..colors import xkcd_rgb, crayons

from distutils.version import LooseVersion
mpl_ge_150 = LooseVersion(mpl.__version__) >= '1.5.0'

class TestColorPalettes(object):

    def test_current_palette(self):

        pal = palettes.color_palette(["red", "blue", "green"])
        rcmod.set_palette(pal)
        assert pal == utils.get_color_cycle()
        rcmod.set()

    def test_palette_context(self):

        default_pal = palettes.color_palette()
        context_pal = palettes.color_palette("muted")

        with palettes.color_palette(context_pal):
            nt.assert_equal(utils.get_color_cycle(), context_pal)

        nt.assert_equal(utils.get_color_cycle(), default_pal)

    def test_big_palette_context(self):

        original_pal = palettes.color_palette("deep", n_colors=8)
        context_pal = palettes.color_palette("husl", 10)

        rcmod.set_palette(original_pal)
        with palettes.color_palette(context_pal, 10):
            nt.assert_equal(utils.get_color_cycle(), context_pal)

        nt.assert_equal(utils.get_color_cycle(), original_pal)

        # Reset default
        rcmod.set()

    def test_palette_size(self):

        pal = palettes.color_palette("deep")
        assert len(pal) == palettes.QUAL_PALETTE_SIZES["deep"]

        pal = palettes.color_palette("pastel6")
        assert len(pal) == palettes.QUAL_PALETTE_SIZES["pastel6"]

        pal = palettes.color_palette("Set3")
        assert len(pal) == palettes.QUAL_PALETTE_SIZES["Set3"]

        pal = palettes.color_palette("husl")
        assert len(pal) == 6

        pal = palettes.color_palette("Greens")
        assert len(pal) == 6

    def test_seaborn_palettes(self):

        pals = "deep", "muted", "pastel", "bright", "dark", "colorblind"
        for name in pals:
            full = palettes.color_palette(name, 10).as_hex()
            short = palettes.color_palette(name + "6", 6).as_hex()
            b, _, g, r, m, _, _, _, y, c = full
            assert [b, g, r, m, y, c] == list(short)

    def test_hls_palette(self):

        hls_pal1 = palettes.hls_palette()
        hls_pal2 = palettes.color_palette("hls")
        npt.assert_array_equal(hls_pal1, hls_pal2)

    def test_husl_palette(self):

        husl_pal1 = palettes.husl_palette()
        husl_pal2 = palettes.color_palette("husl")
        npt.assert_array_equal(husl_pal1, husl_pal2)

    def test_mpl_palette(self):

        mpl_pal1 = palettes.mpl_palette("Reds")
        mpl_pal2 = palettes.color_palette("Reds")
        npt.assert_array_equal(mpl_pal1, mpl_pal2)

    def test_mpl_dark_palette(self):

        mpl_pal1 = palettes.mpl_palette("Blues_d")
        mpl_pal2 = palettes.color_palette("Blues_d")
        npt.assert_array_equal(mpl_pal1, mpl_pal2)

    def test_bad_palette_name(self):

        with nt.assert_raises(ValueError):
            palettes.color_palette("IAmNotAPalette")

    def test_terrible_palette_name(self):

        with nt.assert_raises(ValueError):
            palettes.color_palette("jet")

    def test_bad_palette_colors(self):

        pal = ["red", "blue", "iamnotacolor"]
        with nt.assert_raises(ValueError):
            palettes.color_palette(pal)

    def test_palette_desat(self):

        pal1 = palettes.husl_palette(6)
        pal1 = [utils.desaturate(c, .5) for c in pal1]
        pal2 = palettes.color_palette("husl", desat=.5)
        npt.assert_array_equal(pal1, pal2)

    def test_palette_is_list_of_tuples(self):

        pal_in = np.array(["red", "blue", "green"])
        pal_out = palettes.color_palette(pal_in, 3)

        nt.assert_is_instance(pal_out, list)
        nt.assert_is_instance(pal_out[0], tuple)
        nt.assert_is_instance(pal_out[0][0], float)
        nt.assert_equal(len(pal_out[0]), 3)

    def test_palette_cycles(self):

        deep = palettes.color_palette("deep6")
        double_deep = palettes.color_palette("deep6", 12)
        nt.assert_equal(double_deep, deep + deep)

    def test_hls_values(self):

        pal1 = palettes.hls_palette(6, h=0)
        pal2 = palettes.hls_palette(6, h=.5)
        pal2 = pal2[3:] + pal2[:3]
        npt.assert_array_almost_equal(pal1, pal2)

        pal_dark = palettes.hls_palette(5, l=.2)  # noqa
        pal_bright = palettes.hls_palette(5, l=.8)  # noqa
        npt.assert_array_less(list(map(sum, pal_dark)),
                              list(map(sum, pal_bright)))

        pal_flat = palettes.hls_palette(5, s=.1)
        pal_bold = palettes.hls_palette(5, s=.9)
        npt.assert_array_less(list(map(np.std, pal_flat)),
                              list(map(np.std, pal_bold)))

    def test_husl_values(self):

        pal1 = palettes.husl_palette(6, h=0)
        pal2 = palettes.husl_palette(6, h=.5)
        pal2 = pal2[3:] + pal2[:3]
        npt.assert_array_almost_equal(pal1, pal2)

        pal_dark = palettes.husl_palette(5, l=.2)  # noqa
        pal_bright = palettes.husl_palette(5, l=.8)  # noqa
        npt.assert_array_less(list(map(sum, pal_dark)),
                              list(map(sum, pal_bright)))

        pal_flat = palettes.husl_palette(5, s=.1)
        pal_bold = palettes.husl_palette(5, s=.9)
        npt.assert_array_less(list(map(np.std, pal_flat)),
                              list(map(np.std, pal_bold)))

    def test_cbrewer_qual(self):

        pal_short = palettes.mpl_palette("Set1", 4)
        pal_long = palettes.mpl_palette("Set1", 6)
        nt.assert_equal(pal_short, pal_long[:4])

        pal_full = palettes.mpl_palette("Set2", 8)
        pal_long = palettes.mpl_palette("Set2", 10)
        nt.assert_equal(pal_full, pal_long[:8])

    def test_mpl_reversal(self):

        pal_forward = palettes.mpl_palette("BuPu", 6)
        pal_reverse = palettes.mpl_palette("BuPu_r", 6)
        npt.assert_array_almost_equal(pal_forward, pal_reverse[::-1])

    def test_rgb_from_hls(self):

        color = .5, .8, .4
        rgb_got = palettes._color_to_rgb(color, "hls")
        rgb_want = colorsys.hls_to_rgb(*color)
        nt.assert_equal(rgb_got, rgb_want)

    def test_rgb_from_husl(self):

        color = 120, 50, 40
        rgb_got = palettes._color_to_rgb(color, "husl")
        rgb_want = husl.husl_to_rgb(*color)
        nt.assert_equal(rgb_got, rgb_want)

    def test_rgb_from_xkcd(self):

        color = "dull red"
        rgb_got = palettes._color_to_rgb(color, "xkcd")
        rgb_want = xkcd_rgb[color]
        nt.assert_equal(rgb_got, rgb_want)

    def test_light_palette(self):

        pal_forward = palettes.light_palette("red")
        pal_reverse = palettes.light_palette("red", reverse=True)
        npt.assert_array_almost_equal(pal_forward, pal_reverse[::-1])

        red = tuple(mpl.colors.colorConverter.to_rgba("red"))
        nt.assert_equal(tuple(pal_forward[-1]), red)

        pal_cmap = palettes.light_palette("blue", as_cmap=True)
        nt.assert_is_instance(pal_cmap, mpl.colors.LinearSegmentedColormap)

    def test_dark_palette(self):

        pal_forward = palettes.dark_palette("red")
        pal_reverse = palettes.dark_palette("red", reverse=True)
        npt.assert_array_almost_equal(pal_forward, pal_reverse[::-1])

        red = tuple(mpl.colors.colorConverter.to_rgba("red"))
        nt.assert_equal(tuple(pal_forward[-1]), red)

        pal_cmap = palettes.dark_palette("blue", as_cmap=True)
        nt.assert_is_instance(pal_cmap, mpl.colors.LinearSegmentedColormap)

    def test_blend_palette(self):

        colors = ["red", "yellow", "white"]
        pal_cmap = palettes.blend_palette(colors, as_cmap=True)
        nt.assert_is_instance(pal_cmap, mpl.colors.LinearSegmentedColormap)

    def test_cubehelix_against_matplotlib(self):

        x = np.linspace(0, 1, 8)
        mpl_pal = mpl.cm.cubehelix(x)[:, :3].tolist()

        sns_pal = palettes.cubehelix_palette(8, start=0.5, rot=-1.5, hue=1,
                                             dark=0, light=1, reverse=True)

        nt.assert_list_equal(sns_pal, mpl_pal)

    def test_cubehelix_n_colors(self):

        for n in [3, 5, 8]:
            pal = palettes.cubehelix_palette(n)
            nt.assert_equal(len(pal), n)

    def test_cubehelix_reverse(self):

        pal_forward = palettes.cubehelix_palette()
        pal_reverse = palettes.cubehelix_palette(reverse=True)
        nt.assert_list_equal(pal_forward, pal_reverse[::-1])

    def test_cubehelix_cmap(self):

        cmap = palettes.cubehelix_palette(as_cmap=True)
        nt.assert_is_instance(cmap, mpl.colors.ListedColormap)
        pal = palettes.cubehelix_palette()
        x = np.linspace(0, 1, 6)
        npt.assert_array_equal(cmap(x)[:, :3], pal)

        cmap_rev = palettes.cubehelix_palette(as_cmap=True, reverse=True)
        x = np.linspace(0, 1, 6)
        pal_forward = cmap(x).tolist()
        pal_reverse = cmap_rev(x[::-1]).tolist()
        nt.assert_list_equal(pal_forward, pal_reverse)

    def test_cubehelix_code(self):

        color_palette = palettes.color_palette
        cubehelix_palette = palettes.cubehelix_palette

        pal1 = color_palette("ch:", 8)
        pal2 = color_palette(cubehelix_palette(8))
        assert pal1 == pal2

        pal1 = color_palette("ch:.5, -.25,hue = .5,light=.75", 8)
        pal2 = color_palette(cubehelix_palette(8, .5, -.25, hue=.5, light=.75))
        assert pal1 == pal2

        pal1 = color_palette("ch:h=1,r=.5", 9)
        pal2 = color_palette(cubehelix_palette(9, hue=1, rot=.5))
        assert pal1 == pal2

        pal1 = color_palette("ch:_r", 6)
        pal2 = color_palette(cubehelix_palette(6, reverse=True))
        assert pal1 == pal2

    def test_xkcd_palette(self):

        names = list(xkcd_rgb.keys())[10:15]
        colors = palettes.xkcd_palette(names)
        for name, color in zip(names, colors):
            as_hex = mpl.colors.rgb2hex(color)
            nt.assert_equal(as_hex, xkcd_rgb[name])

    def test_crayon_palette(self):

        names = list(crayons.keys())[10:15]
        colors = palettes.crayon_palette(names)
        for name, color in zip(names, colors):
            as_hex = mpl.colors.rgb2hex(color)
            nt.assert_equal(as_hex, crayons[name].lower())

    def test_color_codes(self):

        palettes.set_color_codes("deep")
        colors = palettes.color_palette("deep6") + [".1"]
        for code, color in zip("bgrmyck", colors):
            rgb_want = mpl.colors.colorConverter.to_rgb(color)
            rgb_got = mpl.colors.colorConverter.to_rgb(code)
            nt.assert_equal(rgb_want, rgb_got)
        palettes.set_color_codes("reset")

        with pytest.raises(ValueError):
            palettes.set_color_codes("Set1")

    def test_as_hex(self):

        pal = palettes.color_palette("deep")
        for rgb, hex in zip(pal, pal.as_hex()):
            nt.assert_equal(mpl.colors.rgb2hex(rgb), hex)

    def test_preserved_palette_length(self):

        pal_in = palettes.color_palette("Set1", 10)
        pal_out = palettes.color_palette(pal_in)
        nt.assert_equal(pal_in, pal_out)

    def test_get_color_cycle(self):

        if mpl_ge_150:
            colors = [(1., 0., 0.), (0, 1., 0.)]
            prop_cycle = plt.cycler(color=colors)
            with plt.rc_context({"axes.prop_cycle": prop_cycle}):
                result = utils.get_color_cycle()
                assert result == colors


#-----------------------------------------------------------------------------
# Copyright (c) 2012 - 2017, Anaconda, Inc. All rights reserved.
#
# Powered by the Bokeh Development Team.
#
# The full license is in the file LICENSE.txt, distributed with this software.
#-----------------------------------------------------------------------------
''' Helper functions for downloading and accessing sample data.

'''

#-----------------------------------------------------------------------------
# Boilerplate
#-----------------------------------------------------------------------------
from __future__ import absolute_import, division, print_function, unicode_literals

import logging
log = logging.getLogger(__name__)

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Standard library imports
from os import mkdir, remove
from os.path import abspath, dirname, exists, expanduser, isdir, isfile, join, splitext
from sys import stdout
from zipfile import ZipFile

# External imports
import six
from six.moves.urllib.request import urlopen

# Bokeh imports
from .dependencies import import_required

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

__all__ = (
    'download',
)

#-----------------------------------------------------------------------------
# General API
#-----------------------------------------------------------------------------

def download(progress=True):
    ''' Download larger data sets for various Bokeh examples.

    '''
    data_dir = external_data_dir(create=True)
    print("Using data directory: %s" % data_dir)

    s3 = 'https://s3.amazonaws.com/bokeh_data/'
    files = [
        (s3, 'CGM.csv'),
        (s3, 'US_Counties.zip'),
        (s3, 'us_cities.json'),
        (s3, 'unemployment09.csv'),
        (s3, 'AAPL.csv'),
        (s3, 'FB.csv'),
        (s3, 'GOOG.csv'),
        (s3, 'IBM.csv'),
        (s3, 'MSFT.csv'),
        (s3, 'WPP2012_SA_DB03_POPULATION_QUINQUENNIAL.zip'),
        (s3, 'gapminder_fertility.csv'),
        (s3, 'gapminder_population.csv'),
        (s3, 'gapminder_life_expectancy.csv'),
        (s3, 'gapminder_regions.csv'),
        (s3, 'world_cities.zip'),
        (s3, 'airports.json'),
        (s3, 'movies.db.zip'),
        (s3, 'airports.csv'),
        (s3, 'routes.csv'),
    ]

    for base_url, filename in files:
        _download_file(base_url, filename, data_dir, progress=progress)

#-----------------------------------------------------------------------------
# Dev API
#-----------------------------------------------------------------------------

def external_csv(module, name, **kw):
    '''

    '''
    pd = import_required('pandas', '%s sample data requires Pandas (http://pandas.pydata.org) to be installed' % module)
    return pd.read_csv(external_path(name), **kw)

def external_data_dir(create=False):
    '''

    '''
    try:
        import yaml
    except ImportError:
        raise RuntimeError("'yaml' and 'pyyaml' are required to use bokeh.sampledata functions")

    bokeh_dir = _bokeh_dir(create=create)
    data_dir = join(bokeh_dir, "data")

    try:
        config = yaml.load(open(join(bokeh_dir, 'config')))
        data_dir = expanduser(config['sampledata_dir'])
    except (IOError, TypeError):
        pass

    if not exists(data_dir):
        if not create:
            raise RuntimeError('bokeh sample data directory does not exist, please execute bokeh.sampledata.download()')
        print("Creating %s directory" % data_dir)
        try:
            mkdir(data_dir)
        except OSError:
            raise RuntimeError("could not create bokeh data directory at %s" % data_dir)
    else:
        if not isdir(data_dir):
            raise RuntimeError("%s exists but is not a directory" % data_dir)

    return data_dir

def external_path(filename):
    data_dir = external_data_dir()
    fn = join(data_dir, filename)
    if not exists(fn) and isfile(fn):
        raise RuntimeError('Could not locate external data file %e. Please execute bokeh.sampledata.download()' % fn)
    return fn

def package_csv(module, name, **kw):
    '''

    '''
    pd = import_required('pandas', '%s sample data requires Pandas (http://pandas.pydata.org) to be installed' % module)
    return pd.read_csv(package_path(name), **kw)

def package_dir():
    '''

    '''
    return abspath(join(dirname(__file__), "..", "sampledata", "_data"))

def package_path(filename):
    '''

    '''
    return join(package_dir(), filename)

def open_csv(filename):
    '''

    '''
    # csv differs in Python 2.x and Python 3.x. Open the file differently in each.
    if six.PY2:
        return open(filename, 'rb')
    else:
        return open(filename, 'r', newline='', encoding='utf8')

#-----------------------------------------------------------------------------
# Private API
#-----------------------------------------------------------------------------

def _bokeh_dir(create=False):
    '''

    '''
    bokeh_dir = join(expanduser("~"), ".bokeh")
    if not exists(bokeh_dir):
        if not create: return bokeh_dir
        print("Creating %s directory" % bokeh_dir)
        try:
            mkdir(bokeh_dir)
        except OSError:
            raise RuntimeError("could not create bokeh config directory at %s" % bokeh_dir)
    else:
        if not isdir(bokeh_dir):
            raise RuntimeError("%s exists but is not a directory" % bokeh_dir)
    return bokeh_dir

def _download_file(base_url, filename, data_dir, progress=True):
    '''

    '''
    file_url = join(base_url, filename)
    file_path = join(data_dir, filename)

    url = urlopen(file_url)

    with open(file_path, 'wb') as file:
        file_size = int(url.headers["Content-Length"])
        print("Downloading: %s (%d bytes)" % (filename, file_size))

        fetch_size = 0
        block_size = 16384

        while True:
            data = url.read(block_size)
            if not data:
                break

            fetch_size += len(data)
            file.write(data)

            if progress:
                status = "\r%10d [%6.2f%%]" % (fetch_size, fetch_size*100.0/file_size)
                stdout.write(status)
                stdout.flush()

    if progress:
        print()

    real_name, ext = splitext(filename)

    if ext == '.zip':
        if not splitext(real_name)[1]:
            real_name += ".csv"

        print("Unpacking: %s" % real_name)

        with ZipFile(file_path, 'r') as zip_file:
            zip_file.extract(real_name, data_dir)

        remove(file_path)

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------