from typing import List, Optional

import pandas as pd
import requests
from bs4 import BeautifulSoup, Comment, PageElement, ResultSet

from . import cache
from .utils import most_recent_season

def get_soup(year: int) -> BeautifulSoup:
    url = f'http://www.baseball-reference.com/leagues/MLB/{year}-standings.shtml'
    s = requests.get(url).content
    return BeautifulSoup(s, "lxml")

def get_tables(soup: BeautifulSoup, season: int) -> List[pd.DataFrame]:
    datasets = []
    if season >= 1969:
        tables: List[PageElement] = soup.find_all('table')
        if season == 1981:
            # For some reason BRef has 1981 broken down by halves and overall
            # https://www.baseball-reference.com/leagues/MLB/1981-standings.shtml
            tables = [x for x in tables if 'overall' in x.get('id', '')]
        for table in tables:
            data = []
            headings: List[PageElement] = [th.get_text() for th in table.find("tr").find_all("th")]
            data.append(headings)
            table_body: PageElement = table.find('tbody')
            rows: List[PageElement] = table_body.find_all('tr')
            for row in rows:
                cols: List[PageElement] = row.find_all('td')
                cols_text: List[str] = [ele.text.strip() for ele in cols]
                cols_text.insert(0, row.find_all('a')[0].text.strip()) # team name
                data.append([ele for ele in cols_text if ele])
            datasets.append(data)
    else:
        data = []
        table = soup.find('table')
        headings = [th.get_text() for th in table.find("tr").find_all("th")]
        headings[0] = "Name"
        if season >= 1930:
            for _ in range(15):
                headings.pop()
        elif season >= 1876:
            for _ in range(14):
                headings.pop()
        else:
            for _ in range(16):
                headings.pop()
        data.append(headings)
        table_body = table.find('tbody')
        rows = table_body.find_all('tr')
        for row in rows:
            if row.find_all('a') == []:
                continue
            cols = row.find_all('td')
            if season >= 1930:
                for _ in range(15):
                    cols.pop()
            elif season >= 1876:
                for _ in range(14):
                    cols.pop()
            else:
                for _ in range(16):
                    cols.pop()
            cols = [ele.text.strip() for ele in cols]
            cols.insert(0,row.find_all('a')[0].text.strip()) # team name
            data.append([ele for ele in cols if ele])
        datasets.append(data)
    #convert list-of-lists to dataframes
    for idx in range(len(datasets)):
        datasets[idx] = pd.DataFrame(datasets[idx])
    return datasets #returns a list of dataframes

@cache.df_cache()
def standings(season:Optional[int] = None) -> pd.DataFrame:
    # get most recent standings if date not specified
    if season is None:
        season = most_recent_season()
    if season < 1876:
        raise ValueError(
            "This query currently only returns standings until the 1876 season. "
            "Try looking at years from 1876 to present."
        )

    # retrieve html from baseball reference
    soup = get_soup(season)
    if season >= 1969:
        raw_tables = get_tables(soup, season)
    else:
        t = [x for x in soup.find_all(string=lambda text:isinstance(text,Comment)) if 'expanded_standings_overall' in x]
        code = BeautifulSoup(t[0], "lxml")
        raw_tables = get_tables(code, season)
    tables = [pd.DataFrame(table) for table in raw_tables]
    for idx in range(len(tables)):
        tables[idx] = tables[idx].rename(columns=tables[idx].iloc[0])
        tables[idx] = tables[idx].reindex(tables[idx].index.drop(0))
    return tables


#!/usr/bin/env python
#-*- coding:utf-8 -*-
"""This is a class for Multinomial Logit Regression

Class uses scipy.optimize package for minimalization of a cost function.
The gradient of the cost function is passed to the minimizer.

Piotr Milanowski, November 2011, Warsaw
"""

from scipy.optimize import fmin_ncg, fmin_bfgs, fmin
import numpy as np
import matplotlib.pyplot as plt

def mix(x1, x2, deg=6):
    out = np.zeros([len(x1), sum(range(deg+2))])
    k = 0
    for i in xrange(deg+1):
        for j in range(i+1):
            out[:,k] = x1**(i-j)*x2**(j)
            k += 1

    return out

class logit(object):
    """This is a class for a normal two-class logistic regression

    The hypothesis of this regression is a sigmoid (logistic, logit) function.
    It returns the probability of the data belonging to the first class.

    The minimalization of a cost function is based on NCG algorithm from scipy.optimize package.

    The regression can account the regularization factors.
    """
    def __init__(self, data, classes, labels=None):
        """Initialization of data

        A column of ones is added to the data array.

        Parameters:
        ===========
        data : 2darray
            NxM array. Rows of this array represent data points, columns represent features.
        classes : 1darray
            a N dimensional vector of classes. Each class is represented by either 0 or 1.
        class_dict [= None] : dictionary
            a 2 element dictionary that maps classses to their names.

        Example:
        =========
            >>>X = np.random.rand(20, 4)     #data
            >>>Y = np.random.randint(0,2,20) #classes
            >>>labels = ['class 1','class 2']
            >>>MLogit.logit(X, Y, labels)
        """
        self.dataNo, self.featureNo = data.shape
        if len(classes) != self.dataNo:
            raise ValueError, 'Not every data point has its target lable!'
        #Adding a columns of 1s and normalizing data - NO NORMALIZATION NEEDED
        self.X = np.concatenate((np.ones([self.dataNo, 1]), data), axis = 1)
        self.Y = classes

    def _sigmoid(self, z):
        """This returns the value of a sigmoid function.

        Sigmoid/Logistic/Logit finction looks like this:
            f(z) = over{1}{1 + exp(-z)}

        Parameters:
        ===========
        z : ndarray
            the parameter of the function
        Returns:
        sig : ndarray
            values of sigmoid function at z
        """
        return 1/(1 + np.exp(-z))

    def cost_function(self, theta, reg = 0):
        """The cost function of logit regression model

        It looks like this:
            J(theta) = -((1/M)*sum_{i=1}^{M}(y_i*log(h(theta;x_i))+(1-y_i)*log(1-h(theta;x_i)))) +
                        + (reg/2*m)sum_{i=1}^{N}(theta_i)^2
        Parameters:
        ===========
        theta : 1darray
            the array of parameters. It's a (N+1) dimensional vector
        reg [= 0] : float
            the regularization parameter. This parameter penalizes theta being too big (overfitting)
        Returns:
        ========
        J : float
            the value of cost function for given theta
        """

        z = self._sigmoid(np.dot(self.X, theta))
        regular = (reg/(2.0*self.dataNo))*sum(theta[1:]*theta[1:])
        J = self.Y * np.log(z) + (1 - self.Y)*np.log(1 - z)
        J = -(1.0 / self.dataNo) * sum(J)
        return regular + J

    def gradient_function(self, theta, reg = 0):
        """The gradient of cost function

        The gradient looks like this:
            g[0] = 1/N * sum_{i=1}^{N}(h(theta;x_i) - y_i)*x_i^0
            g[j] = 1/N * sum_{i=1}^{N}(h(theta;x_i) - y_i)*x_i^j - theta[j]*reg/N
        Parameters:
        ===========
        theta : 1darray
            the vector of parameters
        reg : float
            the regularization parameter
        Returns:
        ========
        fprime : 1darray
            the gradient of cost function.
        """
        gradient = np.zeros(self.featureNo + 1)
        N = 1.0 / self.dataNo
        z = np.dot(self.X, theta)
        cost = self._sigmoid(z) - self.Y
#        gradient[0] = N * sum(cost * self.X[:, 0])
#        for j in xrange(self.featureNo):
#            gradient[j] = N * sum(cost * self.X[:, j]) - reg * N * theta[j]
        gradient = N * np.dot(cost, self.X)
        gradient[1:] += reg * N *  theta[1:]
        return gradient

    def fit(self, maxiter, reg = 0, initial_gues = None):
        """Minimizing function

        Based on NCG function from scipy.optimize package

        Parameters:
        ===========
        maxiter : int
            maximal number of iterations
        reg [= 0] : float
            regularization parameter
        initial_gueas [= None] : 1darray
            a vector of #features + 1 size. If None zeros will be asumed.
        Returns:
        ========
        theta : 1darray
            optimal model parameters
        """
        if initial_gues is None:
            initial_gues = np.zeros(self.featureNo + 1)

        out = fmin_bfgs(self.cost_function, initial_gues, \
                self.gradient_function, args = ([reg]))
        self.theta = out
        return out

    def predict(self, x, val=0.9):
        """For prediction of x

        Returns predicted probability of x being in class 1
        """
        x = np.insert(x, 0, 1) #inserting one at the beginning
        z = np.dot(x, self.theta)
        #if self._sigmoid(z) >=val:
            #return 1
        #else:
            #return 0
        return self._sigmoid(z)

    def plot_features(self, show=True):
        y = self.Y
        idx = np.argsort(y)
        x = self.X[idx, :]
        y = y[idx]
        N, feats = x.shape
        if feats == 3:
            idx1 = np.where(y==1)[0][0]
            x1 = x[:idx1, :]
            x2 = x[idx1:, :]
            plt.plot(x1[:,1],x1[:,2],'ro',x2[:,1],x2[:,2],'go')
            for x in np.arange(-5, 5, 0.5):
                for y in np.arange(-3, 3, 0.5):
                    if self.predict(np.array([x,y])) <=0.5:
                        plt.plot(x,y,'r+')
                    else:
                        plt.plot(x,y,'g+')
            plt.legend(('Class 0','Class 1'))
            if show:
                plt.show()
        elif feats == 2:
            idx1 = np.where(y==1)[0][0]
            x1 = x[:idx1, :]
            x2 = x[idx1:, :]
            for x in np.arange(x1.min(), x1.max(), 0.1):
                for y in np.arange(x2.min(), x2.max(), 0.1):
                    if self.predict(np.array([x,y])) <=0.01:
                        plt.plot(x,y,'r+')
                    else:
                        plt.plot(x,y,'g+')
            plt.plot(x1[:,1],'ro',x2[:,1],'go')
            if show:
                plt.show()
        else:
            print "More than 2 dimmensions",x.shape

#    def plot_fitted(self):
#        N, feats = self.X.shape
#        if feats == 3:
#            x1 = se

    def __normalization(self, data):
        """Function normalizes the data

        Normalization is done by subtracting the mean of each column from each column member
        and dividing by the column variance.

        Parameters:
        ===========
        data : 2darray
            the data array
        Returns:
        ========
        norms : 2darray
            normalized values
        """
        mean = data.mean(axis = 0)
        variance = data.std(axis = 0)
        return (data - mean) / variance

class mlogit(logit):
    """This is a multivariate variation of logit model

    """
    def __init__(self, data, classes, labels=None):
        """See logit description"""
        super(mlogit, self).__init__(data, classes, labels)
        self.classesNo, classesIdx = np.unique(classes, return_inverse = True)
        self.count_table = np.zeros([len(classes), len(self.classesNo)])
        self.count_table[range(len(classes)), classesIdx] = 1.0

    def fit(self, maxiter, reg = 0, initial_gues = None):
        """Fitting logit model for multiclass case"""
        theta = np.zeros([self.featureNo + 1, len(self.classesNo)])
        for i in range(len(self.classesNo)):
            self.Y = self.count_table[:,i]
            theta[:, i] = super(mlogit, self).fit(maxiter, reg = reg, initial_gues = initial_gues)
        self.theta = theta
        return theta

    def predict(self, x, val=0.9):
        """Class prediction"""
        x = np.insert(x, 0, 1)
        z = np.dot(x, self.theta)
        probs = super(mlogit, self)._sigmoid(z)
        idx = np.argmax(probs)
        if probs[idx] >= val:
            return self.classesNo[idx]
        else:
            return None

    def plot_features(self):
        cn = len(self.classesNo)
        idx = np.argsort(self.Y)
        y = self.Y[idx]
        x = self.X[idx,:]
        classes = []
        if x.shape[1] == 3:
            for i in range(cn):
                beg, end = np.where(y==i)[0][[0,-1]]
                plt.plot(x[beg:end+1, 1], x[beg:end +1, 2],'o')
                classes.append('Class'+str(i))
            plt.legend(classes)
            plt.show()
        else:
            print "More than 2 dimmesions"

#class diagnostics(object):

#    def __init__(self, classifier_obj, division=[0.6, 0.2, 0.2]):
#        self.obj = classifier_obj
#        self.div = division
#        self.N, self.ft = self.obj.dataNo, self.obj.featureNo
#        self.cvNo = self.N * division[1]
#        self.testNo = self.N * division[2]
#        self.trainNo = self.N * division[0]

#    def diagnose(self, iters, reg, odrer=1, val=0.9):
#        idx = np.linspace(0, self.N-1, self.N)
#        TP, FP, TN, FN
#        train_ok = {'tp':0,'fp':0,'fn':0,'fp':0}
#        cv_ok = {'tp':0,'fp':0,'fn':0,'fp':0}
#        test_ok = {'tp':0,'fp':0,'fn':0,'fp':0}
#        X = self.obj.X
#        Y = self.obj.Y
#        for i in xrange(iters):
#            np.random.shuffle(idx)
#            train_set = X[idx[:self.trainNo], :]
#            cv_set = X[idx[self.trainNo:self.trainNo+self.cvNo], :]
#            test_set = X[idx[self.trainNo+self.cvNo:], :]
#            classes_train = Y[idx[:self.trainNo], :]
#            classes_cv = Y[idx[self.trainNo:self.trainNo+self.cvNo], :]
#            classes_test = Y[idx[self.trainNo+self.cvNo:], :]
#            Training
#            self.obj.X = train_set
#            self.obj.Y = classes_train
#            self.obj.fit(100)
#            for j, row in enumerate(train_set):
#                cl = self.obj.predict(row, val)
#                if cl == classes_train[j]:
#                    train_ok['tp'] += 1
#                elif cl is None:
#                    train_ok['fn'] += 1
#                else:
#                    train_ok['fp'] += 1
#            Crossvalidation
#            for j, row in enumerate(cv_set):
#                cl = self.obj.predict(row, val)
#                if cl == classes_cv[j]:
#                    cv_ok['tp'] += 1
#                elif cl in None:
#                    cv_ok['fn'] += 1
#                else:
#                    cv_ok['fp'] += 1
#            Test set
#            for j, row in enumerate(test_set):
#                cl = self.obj.predict(row, val)
#                if cl == classes_test[j]:
#                    test_ok['tp'] += 1
#                elif cl is None:
#                    test_ok['fn'] += 1
#                else:
#                    test_ok['fp'] += 1

#    def power_set(self, lst, l):
#        """Create a powerset of a list for given length"""
#        r = [[]]
#        for e in lst:
#            r.extend([s + [e] for s in r])
#        return set([j for j in r if len(j) <= l])

#    def next_order(self, kernel, next_o):

#    def make_order(self, p):
#        init_featsNo = self.featNo


#! /usr/bin/env python

"""
Module with post-processing related functions called from within the NFC
algorithm.
"""

__author__ = 'Carlos Alberto Gomez Gonzalez'
__all__ = ['cube_planet_free']

import numpy as np
from ..metrics import cube_inject_companions
import math
from matplotlib.pyplot import plot, xlim, ylim, axes, gca, show

def cube_planet_free(planet_parameter, cube, angs, psfn, plsc, imlib='opencv',
                     interpolation='lanczos4',transmission=None):
    """
    Return a cube in which we have injected negative fake companion at the
    position/flux given by planet_parameter.

    Parameters
    ----------
    planet_parameter: numpy.array or list
        The (r, theta, flux) for all known companions. For a 4d cube r,
        theta and flux must all be 1d arrays with length equal to cube.shape[0];
        i.e. planet_parameter should have shape: (n_pl,3,n_ch).
    cube: numpy.array
        The cube of fits images expressed as a numpy.array.
    angs: numpy.array
        The parallactic angle fits image expressed as a numpy.array.
    psfn: numpy.array
        The scaled psf expressed as a numpy.array.
    plsc: float
        The platescale, in arcsec per pixel.
    imlib : str, optional
        See the documentation of the ``vip_hci.preproc.frame_rotate`` function.
    interpolation : str, optional
        See the documentation of the ``vip_hci.preproc.frame_rotate`` function.

    Returns
    -------
    cpf : numpy.array
        The cube with negative companions injected at the position given in
        planet_parameter.

    """
    cpf = np.zeros_like(cube)

    planet_parameter = np.array(planet_parameter)

    if cube.ndim == 4:
        if planet_parameter.shape[3] != cube.shape[0]:
            raise TypeError("Input planet parameter with wrong dimensions.")

    for i in range(planet_parameter.shape[0]):
        if i == 0:
            cube_temp = cube
        else:
            cube_temp = cpf

        if cube.ndim == 4:
            for j in cube.shape[0]:
                cpf[j] = cube_inject_companions(cube_temp[j], psfn[j], angs,
                                                flevel=-planet_parameter[i, 2, j],
                                                plsc=plsc,
                                                rad_dists=[planet_parameter[i, 0, j]],
                                                n_branches=1,
                                                theta=planet_parameter[i, 1, j],
                                                imlib=imlib,
                                                interpolation=interpolation,
                                                verbose=False,
                                                transmission=transmission)
        else:
            cpf = cube_inject_companions(cube_temp, psfn, angs,
                                         flevel=-planet_parameter[i, 2], plsc=plsc,
                                         rad_dists=[planet_parameter[i, 0]],
                                         n_branches=1, theta=planet_parameter[i, 1],
                                         imlib=imlib, interpolation=interpolation,
                                         verbose=False, transmission=transmission)
    return cpf

def radial_to_eq(r=1, t=0, rError=0, tError=0, display=False):
    """
    Convert the position given in (r,t) into \delta RA and \delta DEC, as
    well as the corresponding uncertainties.
    t = 0 deg (resp. 90 deg) points toward North (resp. East).

    Parameters
    ----------
    r: float
        The radial coordinate.
    t: float
        The angular coordinate.
    rError: float
        The error bar related to r.
    tError: float
        The error bar related to t.
    display: boolean, optional
        If True, a figure illustrating the error ellipse is displayed.

    Returns
    -------
    out : tuple
        ((RA, RA error), (DEC, DEC error))

    """
    ra = (r * np.sin(math.radians(t)))
    dec = (r * np.cos(math.radians(t)))
    u, v = (ra, dec)

    nu = np.mod(np.pi/2-math.radians(t), 2*np.pi)
    a, b = (rError,r*np.sin(math.radians(tError)))

    beta = np.linspace(0, 2*np.pi, 5000)
    x, y = (u + (a * np.cos(beta) * np.cos(nu) - b * np.sin(beta) * np.sin(nu)),
            v + (b * np.sin(beta) * np.cos(nu) + a * np.cos(beta) * np.sin(nu)))

    raErrorInf = u - np.amin(x)
    raErrorSup = np.amax(x) - u
    decErrorInf = v - np.amin(y)
    decErrorSup = np.amax(y) - v

    if display:
        plot(u,v,'ks',x,y,'r')
        plot((r+rError) * np.cos(nu), (r+rError) * np.sin(nu),'ob',
             (r-rError) * np.cos(nu), (r-rError) * np.sin(nu),'ob')
        plot(r * np.cos(nu+math.radians(tError)),
             r*np.sin(nu+math.radians(tError)),'ok')
        plot(r * np.cos(nu-math.radians(tError)),
             r*np.sin(nu-math.radians(tError)),'ok')
        plot(0,0,'og',np.cos(np.linspace(0,2*np.pi,10000)) * r,
             np.sin(np.linspace(0,2*np.pi,10000)) * r,'y')
        plot([0,r*np.cos(nu+math.radians(tError*0))],
             [0,r*np.sin(nu+math.radians(tError*0))],'k')
        axes().set_aspect('equal')
        lim = np.amax([a,b]) * 2.
        xlim([ra-lim,ra+lim])
        ylim([dec-lim,dec+lim])
        gca().invert_xaxis()
        show()

    return ((ra,np.mean([raErrorInf,raErrorSup])),
            (dec,np.mean([decErrorInf,decErrorSup])))

def cart_to_polar(y, x, ceny=0, cenx=0):
    """
    Convert cartesian into polar coordinates (r,theta) with
    respect to a given center (cenx,ceny).

    Parameters
    ----------
    x,y: float
        The cartesian coordinates.

    Returns
    -------

    out : tuple
        The polar coordinates (r,theta) with respect to the (cenx,ceny).
        Note that theta is given in degrees.

    """
    r = np.sqrt((y-ceny)**2 + (x-cenx)**2)
    theta = np.degrees(np.arctan2(y-ceny, x-cenx))

    return r, np.mod(theta,360)

def polar_to_cart(r, theta, ceny=0, cenx=0):
    """
    Convert polar coordinates with respect to the center (cenx,ceny) into
    cartesian coordinates (x,y) with respect to the bottom left corner of the
    image..

    Parameters
    ----------
    r,theta: float
        The polar coordinates.

    Returns
    -------

    out : tuple
        The cartesian coordinates (x,y) with respect to the bottom left corner
        of the image..

    """
    x = r*np.cos(np.deg2rad(theta)) + cenx
    y = r*np.sin(np.deg2rad(theta)) + ceny

    return x,y

def ds9index_to_polar(y, x, ceny=0, cenx=0):
    """
    Convert pixel index read on image displayed with DS9 into polar coordinates
    (r,theta) with respect to a given center (cenx,ceny).

    Note that ds9 index (x,y) = Python matrix index (y,x). Furthermore, when an
    image M is displayed with DS9, the coordinates of the center of the pixel
    associated with M[0,0] is (1,1). Then, there is a shift of (0.5, 0.5) of the
    center of the coordinate system. As a conclusion, when you read
    (x_ds9, y_ds9) on a image displayed with DS9, the corresponding position is
    (y-0.5, x-0.5) and the associated pixel value is
    M(np.floor(y)-1,np.floor(x)-1).

    Parameters
    ----------
    x,y: float
        The pixel index in DS9

    Returns
    -------

    out : tuple
        The polar coordinates (r,theta) with respect to the (cenx,ceny).
        Note that theta is given in degrees.

    """
    r = np.sqrt((y-0.5-ceny)**2 + (x-0.5-cenx)**2)
    theta = np.degrees(np.arctan2(y-0.5-ceny, x-0.5-cenx))

    return r, np.mod(theta,360)

def polar_to_ds9index(r, theta, ceny=0, cenx=0):
    """
    Convert position (r,theta) in an image with respect to a given center
    (cenx,ceny) into position in the image displayed with DS9.

    Note that ds9 index (x,y) = Python matrix index (y,x). Furthermore, when an
    image M is displayed with DS9, the coordinates of the center of the pixel
    associated with M[0,0] is (1,1). Then, there is a shift of (0.5, 0.5) of the
    center of the coordinate system. As a conclusion, when you read
    (x_ds9, y_ds9) on a image displayed with DS9, the corresponding position is
    (y-0.5, x-0.5) and the associated pixel value is
    M(np.floor(y)-1,np.floor(x)-1).

    Parameters
    ----------
    x,y: float
        The pixel index in DS9

    Returns
    -------

    out : tuple
        The polar coordinates (r,theta) with respect to the (cenx,ceny).
        Note that theta is given in degrees.

    """
    x_ds9 = r*np.cos(np.deg2rad(theta)) + 0.5 + cenx
    y_ds9 = r*np.sin(np.deg2rad(theta)) + 0.5 + ceny

    return x_ds9, y_ds9


import pickle
from feature_process import FeatureMapping
import feature_process
from text_processing import TextProcessing
from sklearn.cross_validation import train_test_split

is_not_important = {9:0,
13:0,
18:0,
19:0,
23:0,
28:0,
29:0,
33:0,
34:0,
37:0,
40:0,
44:0,
46:0,
50:0,
55:0,
59:0,
61:0,
62:0,
63:0,
72:0,
73:0,
78:0,
84:0,
86:0,
88:0,
97:0,
98:0,
103:0
}

def create_training_data():
    data_lst = pickle.load(open('data/harvest.data', 'rb'))
    feature_process.feature_map['source'] = {'Google':1, 'Twitter for iPad':2, 'Echofon':3,
                                             'Bitly':4, 'twitterfeed':5, 'Twitter for iPhone':6,
                                             'Foursquare':7, 'Facebook':8, 'Twitter for Android':9,
                                             'TweetDeck':10, 'Twitter Web Client':11}
    feature_process.feature_map['geo'] = ['None']
    feature_process.feature_map['place'] = ['None']
    feature_process.feature_map['verified'] = ['False']
    feature_process.feature_map['geo_enabled'] = ['False']
    y = []
    x = []
    for i in range(0, len(data_lst)):
        try:
            label = is_not_important[i]
        except Exception as e:
            label = 1

        data = data_lst[i]
        text = TextProcessing.process(data[0])
        source = FeatureMapping.mapping('source', data[1])
        re_tweet = data[2]
        geo = FeatureMapping.mapping_other('geo', data[3])
        place = FeatureMapping.mapping_other('place', data[4])
        hash_tag = data[5]
        media = data[6]
        verified = FeatureMapping.mapping_other('verified', data[7])
        follower = data[8]
        statues = data[9]
        desc = TextProcessing.process(data[10])
        friend = data[11]
        location = TextProcessing.process(data[12])
        geo_enabled = FeatureMapping.mapping_other('geo_enabled', data[13])

        y.append(label)
        x.append([text, source, re_tweet, geo, place, hash_tag, media, verified, follower, statues, desc, friend, location, geo_enabled])
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.metrics import f1_score, accuracy_score
    clf = RandomForestClassifier()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)
    fsc = f1_score(y_test, y_pred)
    acc = accuracy_score(y_test, y_pred)
    print 'f1-score : ',fsc
    print 'accuracy : ',acc
    print y_pred
    print y_test

if __name__ == '__main__':
    create_training_data()


"""
================================
SVM Exercise
================================

A tutorial exercise for using different SVM kernels.

This exercise is used in the :ref:`using_kernels_tut` part of the
:ref:`supervised_learning_tut` section of the :ref:`stat_learn_tut_index`.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets, svm

iris = datasets.load_iris()
X = iris.data
y = iris.target

X = X[y != 0, :2]
y = y[y != 0]

n_sample = len(X)

np.random.seed(0)
order = np.random.permutation(n_sample)
X = X[order]
y = y[order].astype(np.float)

X_train = X[:.9 * n_sample]
y_train = y[:.9 * n_sample]
X_test = X[.9 * n_sample:]
y_test = y[.9 * n_sample:]

# fit the model
for fig_num, kernel in enumerate(('linear', 'rbf', 'poly')):
    clf = svm.SVC(kernel=kernel, gamma=10)
    clf.fit(X_train, y_train)

    plt.figure(fig_num)
    plt.clf()
    plt.scatter(X[:, 0], X[:, 1], c=y, zorder=10, cmap=plt.cm.Paired)

    # Circle out the test data
    plt.scatter(X_test[:, 0], X_test[:, 1], s=80, facecolors='none', zorder=10)

    plt.axis('tight')
    x_min = X[:, 0].min()
    x_max = X[:, 0].max()
    y_min = X[:, 1].min()
    y_max = X[:, 1].max()

    XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
    Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(XX.shape)
    plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.Paired)
    plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'],
                levels=[-.5, 0, .5])

    plt.title(kernel)
plt.show()


"""
This is a procedural interface to the matplotlib object-oriented
plotting library.

The following plotting commands are provided; the majority have
Matlab(TM) analogs and similar argument.

_Plotting commands
  acorr     - plot the autocorrelation function
  annotate  - annotate something in the figure
  arrow     - add an arrow to the axes
  axes      - Create a new axes
  axhline   - draw a horizontal line across axes
  axvline   - draw a vertical line across axes
  axhspan   - draw a horizontal bar across axes
  axvspan   - draw a vertical bar across axes
  axis      - Set or return the current axis limits
  bar       - make a bar chart
  barh      - a horizontal bar chart
  broken_barh - a set of horizontal bars with gaps
  box       - set the axes frame on/off state
  boxplot   - make a box and whisker plot
  cla       - clear current axes
  clabel    - label a contour plot
  clf       - clear a figure window
  clim      - adjust the color limits of the current image
  close     - close a figure window
  colorbar  - add a colorbar to the current figure
  cohere    - make a plot of coherence
  contour   - make a contour plot
  contourf  - make a filled contour plot
  csd       - make a plot of cross spectral density
  delaxes   - delete an axes from the current figure
  draw      - Force a redraw of the current figure
  errorbar  - make an errorbar graph
  figlegend - make legend on the figure rather than the axes
  figimage  - make a figure image
  figtext   - add text in figure coords
  figure   - create or change active figure
  fill     - make filled polygons
  findobj  - recursively find all objects matching some criteria
  gca      - return the current axes
  gcf      - return the current figure
  gci      - get the current image, or None
  getp      - get a handle graphics property
  grid     - set whether gridding is on
  hist     - make a histogram
  hold     - set the axes hold state
  ioff     - turn interaction mode off
  ion      - turn interaction mode on
  isinteractive - return True if interaction mode is on
  imread   - load image file into array
  imshow   - plot image data
  ishold   - return the hold state of the current axes
  legend   - make an axes legend
  loglog   - a log log plot
  matshow  - display a matrix in a new figure preserving aspect
  pcolor   - make a pseudocolor plot
  pcolormesh - make a pseudocolor plot using a quadrilateral mesh
  pie      - make a pie chart
  plot     - make a line plot
  plot_date - plot dates
  plotfile  - plot column data from an ASCII tab/space/comma delimited file
  pie      - pie charts
  polar    - make a polar plot on a PolarAxes
  psd      - make a plot of power spectral density
  quiver   - make a direction field (arrows) plot
  rc       - control the default params
  rgrids   - customize the radial grids and labels for polar
  savefig  - save the current figure
  scatter  - make a scatter plot
  setp      - set a handle graphics property
  semilogx - log x axis
  semilogy - log y axis
  show     - show the figures
  specgram - a spectrogram plot
  spy      - plot sparsity pattern using markers or image
  stem     - make a stem plot
  subplot  - make a subplot (numrows, numcols, axesnum)
  subplots_adjust - change the params controlling the subplot positions of current figure
  subplot_tool - launch the subplot configuration tool
  suptitle   - add a figure title
  table    - add a table to the plot
  text     - add some text at location x,y to the current axes
  thetagrids - customize the radial theta grids and labels for polar
  title    - add a title to the current axes
  xcorr   - plot the autocorrelation function of x and y
  xlim     - set/get the xlimits
  ylim     - set/get the ylimits
  xticks   - set/get the xticks
  yticks   - set/get the yticks
  xlabel   - add an xlabel to the current axes
  ylabel   - add a ylabel to the current axes

  autumn - set the default colormap to autumn
  bone   - set the default colormap to bone
  cool   - set the default colormap to cool
  copper - set the default colormap to copper
  flag   - set the default colormap to flag
  gray   - set the default colormap to gray
  hot    - set the default colormap to hot
  hsv    - set the default colormap to hsv
  jet    - set the default colormap to jet
  pink   - set the default colormap to pink
  prism  - set the default colormap to prism
  spring - set the default colormap to spring
  summer - set the default colormap to summer
  winter - set the default colormap to winter
  spectral - set the default colormap to spectral

_Event handling

  connect - register an event handler
  disconnect - remove a connected event handler

_Matrix commands

  cumprod   - the cumulative product along a dimension
  cumsum    - the cumulative sum along a dimension
  detrend   - remove the mean or besdt fit line from an array
  diag      - the k-th diagonal of matrix
  diff      - the n-th differnce of an array
  eig       - the eigenvalues and eigen vectors of v
  eye       - a matrix where the k-th diagonal is ones, else zero
  find      - return the indices where a condition is nonzero
  fliplr    - flip the rows of a matrix up/down
  flipud    - flip the columns of a matrix left/right
  linspace  - a linear spaced vector of N values from min to max inclusive
  logspace  - a log spaced vector of N values from min to max inclusive
  meshgrid  - repeat x and y to make regular matrices
  ones      - an array of ones
  rand      - an array from the uniform distribution [0,1]
  randn     - an array from the normal distribution
  rot90     - rotate matrix k*90 degress counterclockwise
  squeeze   - squeeze an array removing any dimensions of length 1
  tri       - a triangular matrix
  tril      - a lower triangular matrix
  triu      - an upper triangular matrix
  vander    - the Vandermonde matrix of vector x
  svd       - singular value decomposition
  zeros     - a matrix of zeros

_Probability

  levypdf   - The levy probability density function from the char. func.
  normpdf   - The Gaussian probability density function
  rand      - random numbers from the uniform distribution
  randn     - random numbers from the normal distribution

_Statistics

  corrcoef  - correlation coefficient
  cov       - covariance matrix
  amax       - the maximum along dimension m
  mean      - the mean along dimension m
  median    - the median along dimension m
  amin       - the minimum along dimension m
  norm      - the norm of vector x
  prod      - the product along dimension m
  ptp       - the max-min along dimension m
  std       - the standard deviation along dimension m
  asum       - the sum along dimension m

_Time series analysis

  bartlett  - M-point Bartlett window
  blackman  - M-point Blackman window
  cohere    - the coherence using average periodiogram
  csd       - the cross spectral density using average periodiogram
  fft       - the fast Fourier transform of vector x
  hamming   - M-point Hamming window
  hanning   - M-point Hanning window
  hist      - compute the histogram of x
  kaiser    - M length Kaiser window
  psd       - the power spectral density using average periodiogram
  sinc      - the sinc function of array x

_Dates

  date2num  - convert python datetimes to numeric representation
  drange    - create an array of numbers for date plots
  num2date  - convert numeric type (float days since 0001) to datetime

_Other

  angle     - the angle of a complex array
  griddata - interpolate irregularly distributed data to a regular grid
  load     - load ASCII data into array
  polyfit   - fit x, y to an n-th order polynomial
  polyval   - evaluate an n-th order polynomial
  roots     - the roots of the polynomial coefficients in p
  save      - save an array to an ASCII file
  trapz     - trapezoidal integration

__end

"""
import sys, warnings

from cbook import flatten, is_string_like, exception_to_str, popd, \
     silent_list, iterable, dedent

import numpy as np
from numpy import ma

from matplotlib import mpl  # pulls in most modules

from matplotlib.dates import date2num, num2date,\
        datestr2num, strpdate2num, drange,\
        epoch2num, num2epoch, mx2num,\
        DateFormatter, IndexDateFormatter, DateLocator,\
        RRuleLocator, YearLocator, MonthLocator, WeekdayLocator,\
        DayLocator, HourLocator, MinuteLocator, SecondLocator,\
        rrule, MO, TU, WE, TH, FR, SA, SU, YEARLY, MONTHLY,\
        WEEKLY, DAILY, HOURLY, MINUTELY, SECONDLY, relativedelta
import matplotlib.dates

# bring all the  symbols in so folks can import them from
# pylab in one fell swoop

from matplotlib.mlab import  window_hanning, window_none,\
        conv, detrend, detrend_mean, detrend_none, detrend_linear,\
        polyfit, polyval, entropy, normpdf, griddata,\
        levypdf, find, trapz, prepca, rem, norm, orth, rank,\
        sqrtm, prctile, center_matrix, rk4, exp_safe, amap,\
        sum_flat, mean_flat, rms_flat, l1norm, l2norm, norm, frange,\
        diagonal_matrix, base_repr, binary_repr, log2, ispower2,\
        bivariate_normal, load, save

from matplotlib.mlab import stineman_interp, slopes, \
    stineman_interp, inside_poly, poly_below, poly_between, \
    is_closed_polygon, path_length, distances_along_curve, vector_lengths

from numpy import *
from numpy.fft import *
from numpy.random import *
from numpy.linalg import *

from matplotlib.mlab import window_hanning, window_none, conv, detrend, demean, \
     detrend_mean, detrend_none, detrend_linear, entropy, normpdf, levypdf, \
     find, longest_contiguous_ones, longest_ones, prepca, prctile, prctile_rank, \
     center_matrix, rk4, bivariate_normal, get_xyz_where, get_sparse_matrix, dist, \
     dist_point_to_segment, segments_intersect, fftsurr, liaupunov, movavg, \
     save, load, exp_safe, \
     amap, rms_flat, l1norm, l2norm, norm_flat, frange, diagonal_matrix, identity, \
     base_repr, binary_repr, log2, ispower2, fromfunction_kw, rem, norm, orth, rank, sqrtm,\
     mfuncC, approx_real, rec_append_field, rec_drop_fields, rec_join, csv2rec, rec2csv, isvector

from matplotlib.pyplot import *

# provide the recommended module abbrevs in the pylab namespace
import matplotlib.pyplot as plt
import numpy as np


"""
================================================
My own Gaussion Mixture Model for SV genotyping.
Learn form scikit-learn
================================================

Author : Shujia Huang
Date   : 2014-01-06 14:33:45

"""
import sys
import numpy as np
from scipy import linalg
from sklearn import cluster
from sklearn.base import BaseEstimator
from sklearn.utils.extmath import logsumexp

EPS = np.finfo(float).eps

class GMM ( BaseEstimator ) :

    """
    Copy from scikit-learn
    """

    def __init__(self, n_components=1, covariance_type='diag', random_state=None, thresh=1e-2, min_covar=1e-3,
                 n_iter=100, n_init=10, params='wmc', init_params='wmc'):
        self.n_components    = n_components
        self.covariance_type = covariance_type
        self.thresh          = thresh
        self.min_covar       = min_covar
        self.random_state    = random_state
        self.n_iter          = n_iter
        self.n_init          = n_init
        self.params          = params
        self.init_params     = init_params

        self.init_means      = []
        self.init_covars     = []
        self.category        = []   # For genotype

        if not covariance_type in ['spherical', 'tied', 'diag', 'full']:
            raise ValueError( 'Invalid value for covariance_type: %s' % covariance_type )

        if n_init < 1: raise ValueError('GMM estimation requires at least one run')

        self.weights_ = np.ones(self.n_components) / self.n_components

        # flag to indicate exit status of fit() method: converged (True) or
        # n_iter reached (False)
    def score_samples(self, X):
        """Return the per-sample likelihood of the data under the model.

        Compute the log probability of X under the model and
        return the posterior distribution (responsibilities) of each
        mixture component for each element of X.

        Parameters
        ----------
        X: array_like, shape (n_samples, n_features)
            List of n_features-dimensional data points. Each row
            corresponds to a single data point.

        Returns
        -------
        logprob : array_like, shape (n_samples,)
            Log probabilities of each data point in X.

        responsibilities : array_like, shape (n_samples, n_components)
            Posterior probabilities of each mixture component for each
            observation
        """
        X = np.asarray(X)
        if X.ndim == 1:
            X = X[:, np.newaxis]
        if X.size == 0:
            return np.array([]), np.empty((0, self.n_components))
        if X.shape[1] != self.means_.shape[1]:
            raise ValueError('The shape of X  is not compatible with self')

        lpr = (log_multivariate_normal_density(X, self.means_, self.covars_,self.covariance_type)
               + np.log(self.weights_))

        logprob = logsumexp(lpr, axis=1)
        responsibilities = np.exp(lpr - logprob[:, np.newaxis])
        return logprob, responsibilities

    def predict(self, X):
        """
        Predict label for data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        C : array, shape = (n_samples,)
        """
        logprob, responsibilities = self.score_samples(X)
        return responsibilities.argmax(axis=1)

    def predict_proba(self, X):
        """
        Predict posterior probability of data under each Gaussian
        in the model.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]

        Returns
        -------
        responsibilities : array-like, shape = (n_samples, n_components)
            Returns the probability of the sample for each Gaussian
            (state) in the model.
        """
        logprob, responsibilities = self.score_samples(X)
        return responsibilities

    def fit(self, X):
        """
        Copy form scikit-learn: gmm.py
        Estimate model parameters with the expectation-maximization
        algorithm.

        A initialization step is performed before entering the em
        algorithm. If you want to avoid this step, set the keyword
        argument init_params to the empty string '' when creating the
        GMM object. Likewise, if you would like just to do an
        initialization, set n_iter=0.

        Parameters
        ----------
        X : array_like, shape (n, n_features)
            List of n_features-dimensional data points.  Each row
            corresponds to a single data point.
        """

        X = np.asarray(X, dtype=np.float)
        if X.ndim == 1:
            X = X[:, np.newaxis]
        if X.shape[0] < self.n_components:
            raise ValueError(
                'GMM estimation with %s components, but got only %s samples' %
                (self.n_components, X.shape[0]))

        lowest_bias = np.infty

        c1,c2,c3    = '1/1', '0/1', '0/0'
        m1,m2,m3    = 0.001 , 0.5 , 1.0
        v1,v2,v3    = 0.002, 0.002, 0.002
        category    = np.array([ [c1,c2,c3],
                                 [c1,c2], [c1,c3], [c2,c3] ,
                                 [c1]         , [c2]       , [c3] ])
        init_means  = np.array([ [[ m1],[ m2] , [ m3]],
                                 [[ m1],[ m2]], [[m1],[m3]], [[m2],[m3]],
                                 [[m1]]       , [[m2]]     , [[m3]] ])
        init_covars = np.array([ [[[ v1]],[[ v2]],[[ v3]]],
                                 [[[ v1]],[[ v2]]], [[[ v1]],[[ v3]]], [[[ v2]],[[ v3]]],
                                 [[[ v1]]]        , [[[ v2]]]        , [[[ v3]]] ])

        bestCovars, bestMeans, bestWeights, bestConverged, bestCategory = [], [], [], [], []
        for i, (m,v,c) in enumerate( zip(init_means, init_covars, category) ) :

            if i == 0 and self.n_components != 3 : continue
            if i < 4  and self.n_components == 1 : continue
            self.init_means  = np.array(m)
            self.init_covars = np.array(v)
            self.category    = np.array(c)
            best_params,bias = self.training(X)

            if lowest_bias > bias :
                lowest_bias         = bias
                bestCovars    = best_params['covars']
                bestMeans     = best_params['means']
                bestWeights   = best_params['weights']
                bestConverged = best_params['converged']
                bestCategory  = best_params['category']

            if self.n_components == 3 : break
            if self.n_components == 2 and i == 3 : break

        bestWeights = np.tile(1.0 / self.n_components, self.n_components)

        self.covars_     = bestCovars
        self.means_      = bestMeans
        self.weights_    = bestWeights
        self.converged_  = bestConverged
        self.category    = bestCategory

        return self

    ####
    def training(self, X):

        max_log_prob = -np.infty
        lowest_bias  = np.infty

        wmin, wmax = 0.8, 1.2  # Factor intervel [wmin, wmax]
        for w in np.linspace(wmin, wmax, self.n_init):
            if 'm' in self.init_params or not hasattr(self, 'means_'):
                #self.means_ = cluster.KMeans(n_clusters=self.n_components, random_state=self.random_state).fit(X).cluster_centers_
                self.means_  = w * self.init_means

            if 'w' in self.init_params or not hasattr(self, 'weights_'):
                self.weights_= np.tile(1.0 / self.n_components, self.n_components)

            if 'c' in self.init_params or not hasattr(self, 'covars_'):
                """
                cv = np.cov(X.T) + self.min_covar * np.eye(X.shape[1])
                if not cv.shape :
                    cv.shape = (1, 1)
                self.covars_ = distribute_covar_matrix_to_match_covariance_type(cv, self.covariance_type, self.n_components)
                """
                self.covars_ = self.init_covars

            # EM algorithms
            log_likelihood = []
            # reset self.converged_ to False
            self.converged_= False
            for i in range(self.n_iter):
                # Expectation step
                curr_log_likelihood, responsibilities = self.score_samples(X)
                log_likelihood.append(curr_log_likelihood.sum())

                # Check for convergence.
                if i > 0 and abs(log_likelihood[-1] - log_likelihood[-2]) < self.thresh:
                    self.converged_ = True
                    break
                #Maximization step
                self._do_mstep(X, responsibilities, self.params, self.min_covar)

            if   self.n_components == 3:
                curr_bias =(self.means_[0][0]-self.init_means[0][0])+np.abs(self.means_[1][0]-self.init_means[1][0])+(self.init_means[2][0]-self.means_[2][0])
            elif self.n_components == 2:
                curr_bias =np.abs(self.means_[0][0] - self.init_means[0][0]) + np.abs(self.init_means[1][0] - self.means_[1][0])
            elif self.n_components == 1:
                curr_bias =np.abs (self.means_[0][0] - self.init_means[0][0])
            else :
                 print >> sys.stderr, '[ERROR] The companent could only between [1,3]. But yours is ', self.n_components
                 sys.exit(1)

            self.Label2Genotype()
            if w == wmin:
                max_log_prob = log_likelihood[-1]
                best_params  = {'weights':self.weights_,
                                'means':self.means_,
                                'covars':self.covars_,
                                'converged':self.converged_,
                                'category':self.category}
                if self.converged_:
                    lowest_bias = curr_bias

            if self.converged_ and lowest_bias > curr_bias:
                max_log_prob = log_likelihood[-1]
                lowest_bias  = curr_bias
                best_params  = {'weights': self.weights_,
                                'means': self.means_,
                                'covars': self.covars_,
                                'converged': self.converged_,
                                'category':self.category}

        # check the existence of an init param that was not subject to
        # likelihood computation issue.
        if np.isneginf(max_log_prob) and self.n_iter:
            raise RuntimeError(
                "EM algorithm was never able to compute a valid likelihood " +
                "given initial parameters. Try different init parameters "   +
                "(or increasing n_init) or check for degenerate data." )

       # if neendshift :
       #     self.covars_    = tmp_params['covars']
       #     self.means_     = tmp_params['means']
       #     self.weights_   = tmp_params['weights']
       #     self.converged_ = tmp_params['converged']
       #     self.category   = tmp_params['category']

        return best_params, lowest_bias

    def _do_mstep(self, X, responsibilities, params, min_covar=0):
        """
        Perform the Mstep of the EM algorithm and return the class weihgts.
        """
        weights = responsibilities.sum(axis=0)
        weighted_X_sum  = np.dot(responsibilities.T, X)
        inverse_weights = 1.0 / (weights[:, np.newaxis] + 10 * EPS)

        if 'w' in params:
            self.weights_ = (weights / (weights.sum() + 10 * EPS) + EPS)
        if 'm' in params:
            self.means_ = weighted_X_sum * inverse_weights
        if 'c' in params:
            covar_mstep_func = _covar_mstep_funcs[self.covariance_type]
            self.covars_ = covar_mstep_func(self, X, responsibilities, weighted_X_sum, inverse_weights,min_covar)

        return weights

    """
    Here is just for genotyping process
    """
    # Decide the different guassion mu(mean) to seperate the genotype
    def Label2Genotype(self):

        label2genotype = {}
        if self.converged_:

            if len(self.means_) > 3 :
                print >> sys.stderr, 'Do not allow more than 3 components. But you set', len(self.means_)
                sys.exit(1)

            for label,mu in enumerate(self.means_[:,0]):

                best_distance, bestIndx = np.infty, 0
                for i,m in enumerate(self.init_means[:,0]):
                    distance = np.abs(mu - m)
                    if distance < best_distance:
                        bestIndx      = i
                        best_distance = distance

                label2genotype[label] = self.category[bestIndx]

            # Put False if there are more than one 'label' points to the same 'genotype'
            g2c = {v:k for k,v in label2genotype.items()}
            if len(label2genotype) != len(g2c): self.converged_ = False
        else :
            label2genotype = { label: './.' for label in range( self.n_components ) }

        return label2genotype

    def Mendel(self, genotype, sample2col, family):

        ngIndx = []

        m,n,num = 0.0,0.0,0   # m is match; n is not match
        for k,v in family.items():

            #if v[0] not in sample2col or v[1] not in sample2col : continue
            if k not in sample2col or v[0] not in sample2col or v[1] not in sample2col: continue
            if k not in sample2col :
                print >> sys.stderr, 'The sample name is not in vcf file! ', k
                sys.exit(1)

            # c1 is son; c2 and c3 are the parents
            c1,c2,c3 = genotype[ sample2col[k] ], genotype[ sample2col[v[0]] ], genotype[ sample2col[v[1]] ]

            if c1 == './.' or c2 == './.' or c3 == './.': continue
            num += 1;
            ng   = False
            if c2 == c3 :
                if c2 == '0/0' or c2 == '1/1' :
                    if c1 == c2 : m += 1
                    else :
                        n += 1
                        ng = True
                else : # c2 == '0/1' and c3 == '0/1'
                    m += 1
            elif c2 == '0/1' and c3 == '1/1' :

                if c1 == '0/0' :
                    n += 1
                    ng = True
                else : m += 1
            elif c2 == '0/1' and c3 == '0/0' :

                if   c1 == '1/1' :
                    n += 1
                    ng = True
                else : m += 1
            elif c2 == '1/1' and c3 == '0/1' :

                if c1 == '0/0' :
                    n += 1
                    ng = True
                else : m += 1
            elif c2 == '1/1' and c3 == '0/0' :

                if c1 == '1/1' or c1 == '0/0':
                    n += 1
                    ng = True
                else : m += 1
            elif c2 == '0/0' and c3 == '0/1' :

                if c1 == '1/1' :
                    n += 1
                    ng = True
                else : m += 1
            elif c2 == '0/0' and c3 == '1/1' :

                if c1 == '0/0' or c1 == '1/1' :
                    n += 1
                    ng = True
                else : m += 1

            if ng :
                ngIndx.append(sample2col[k])
                ngIndx.append(sample2col[v[0]])
                ngIndx.append(sample2col[v[1]])

        return m,n,num,set(ngIndx)
###
def log_multivariate_normal_density(X, means, covars, covariance_type='full'):
    """
    Log probability for full covariance matrices.
    """
    X = np.asarray(X)
    if X.ndim == 1:
        X = X[:, np.newaxis]
    if X.size == 0:
        return np.array([])
    if X.shape[1] != means.shape[1]:
        raise ValueError('The shape of X  is not compatible with self')

    log_multivariate_normal_density_dict = {
        'full' : _log_multivariate_normal_density_full
    }

    return log_multivariate_normal_density_dict[covariance_type]( X, means, covars )

def _log_multivariate_normal_density_full(X, means, covars, min_covar=1.e-7):
    """
    Log probability for full covariance matrices.
    """
    if hasattr(linalg, 'solve_triangular'):
        # only in scipy since 0.9
        solve_triangular = linalg.solve_triangular
    else:
        # slower, but works
        solve_triangular = linalg.solve
    n_samples, n_dim = X.shape
    nmix = len(means)
    log_prob = np.empty((n_samples, nmix))
    for c, (mu, cv) in enumerate(zip(means, covars)):
        try:
            cv_chol = linalg.cholesky(cv, lower=True)
        except linalg.LinAlgError:
            # The model is most probabily stuck in a component with too
            # few observations, we need to reinitialize this components
            cv_chol = linalg.cholesky(cv + min_covar * np.eye(n_dim),
                                      lower=True)

        cv_log_det = 2 * np.sum(np.log(np.diagonal(cv_chol)))
        cv_sol = solve_triangular(cv_chol, (X - mu).T, lower=True).T
        log_prob[:, c] = - .5 * (np.sum(cv_sol ** 2, axis=1) +
                                 n_dim * np.log(2 * np.pi) + cv_log_det)

    return log_prob

def distribute_covar_matrix_to_match_covariance_type( tied_cv, covariance_type, n_components) :
    """
    Create all the covariance matrices from a given template
    """
    if covariance_type == 'spherical':
        cv = np.tile(tied_cv.mean() * np.ones(tied_cv.shape[1]),
                     (n_components, 1))
    elif covariance_type == 'tied':
        cv = tied_cv
    elif covariance_type == 'diag':
        cv = np.tile(np.diag(tied_cv), (n_components, 1))
    elif covariance_type == 'full':
        cv = np.tile(tied_cv, (n_components, 1, 1))
    else:
        raise ValueError("covariance_type must be one of " +
                         "'spherical', 'tied', 'diag', 'full'")
    return cv

def _covar_mstep_full(gmm, X, responsibilities, weighted_X_sum, norm, min_covar):
    """Performing the covariance M step for full cases"""
    # Eq. 12 from K. Murphy, "Fitting a Conditional Linear Gaussian
    # Distribution"
    n_features = X.shape[1]
    cv = np.empty((gmm.n_components, n_features, n_features))
    for c in range(gmm.n_components):
        post = responsibilities[:, c]
        # Underflow Errors in doing post * X.T are  not important
        np.seterr(under='ignore')
        avg_cv = np.dot(post * X.T, X) / (post.sum() + 10 * EPS)
        mu = gmm.means_[c][np.newaxis]
        cv[c] = (avg_cv - np.dot(mu.T, mu) + min_covar * np.eye(n_features))
    return cv

_covar_mstep_funcs = { 'full': _covar_mstep_full }


from __future__ import absolute_import

from abc import ABCMeta, abstractmethod
import copy
from data_utils import build_k_indices
from learning_model import *
from regularizer import *
from helpers import save_numpy_array
import numpy as np

class Model(object):
    """
    Author: Kaicheng Yu

    Machine learning model engine
    Implement the optimizers
        sgd
        normal equations
        cross-validation of given parameters
    Abstract method:
        __call__        produce the raw prediction, use the latest weight obtained by training
        predict         produce prediction values, could take weight as input
        get_gradient    define gradient here, including the gradient for regularizer
        normalequ       define normal equations

    Support:
        L1, L2 normalization

    Due to the distribution of work, only LogisticRegression is fully tested for
    fitting data, and cross-validation.
    LinearRegression model should also work but not fully tested.

    The goal of this class is not only specific to this learning project, but also for reusable and scalable
    to other problems, models.

    """
    def __init__(self, train_data, validation=None, initial_weight=None,
                 loss_function_name='mse', cal_weight='gradient',
                 regularizer=None, regularizer_p=None):
        """
        Initializer of all learning models.
        :param train_data: training data.
        :param validation_data:
        """
        self.train_x = train_data[1]
        self.train_y = train_data[0]

        self.set_valid(validation)

        ''' Define the progress of history here '''
        self.losses = []
        self.iterations = 0
        self.weights = []
        self.misclass_rate = []

        ''' Define loss, weight calculation, regularizer '''
        self.loss_function = get_loss_function(loss_function_name)
        self.loss_function_name = loss_function_name
        self.calculate_weight = cal_weight
        self.regularizer = Regularizer.get_regularizer(regularizer, regularizer_p)
        self.regularizer_p = regularizer_p

        # Asserting degree
        if len(self.train_x.shape) > 1:
            degree = self.train_x.shape[1]
        else:
            degree = 1

        # Initialize the weight for linear model.
        if initial_weight is not None:
            self.weights.append(initial_weight)
        else:
            self.weights.append(np.random.rand(degree))

    def set_valid(self, validation):
        # Set validation here.
        self.validation = False
        self.valid_x = None
        self.valid_y = None
        self.valid_losses = None
        self.valid_misclass_rate = None
        if validation is not None:
            (valid_y, valid_x) = validation
            self.valid_x = valid_x
            self.valid_y = valid_y
            self.validation = True
            self.valid_losses = []
            self.valid_misclass_rate = []

    @abstractmethod
    def __call__(self, **kwargs):
        """Define the fit function and get prediction"""
        raise NotImplementedError

    @abstractmethod
    def get_gradient(self, y, x, weight):
        raise NotImplementedError

    @abstractmethod
    def predict(self, x, weight):
        raise NotImplementedError

    @abstractmethod
    def normalequ(self, **kwargs):
        ''' define normal equation method to calculate optimal weights'''
        raise NotImplementedError

    def compute_weight(self, y, x, test_x=None, test_y=None, **kwargs):
        """ Return weight under given parameter """
        model = copy.copy(self)
        model.__setattr__('train_y', y)
        model.__setattr__('train_x', x)
        if test_x is not None and test_y is not None:
            model.set_valid((test_y, test_x))
        _kwargs = []
        for name, value in kwargs.items():
            # Recognize parameter "
            if name is "regularizer_p":
                model.__setattr__(name, value)
                model.regularizer.set_parameter(value)
            else:
                _kwargs.append((name, value))
        _kwargs = dict(_kwargs)
        if model.calculate_weight is 'gradient':
            return model.sgd(**_kwargs)
        # elif model.calculate_weight is 'newton':
        #     return model.newton(**_kwargs)
        elif model.calculate_weight is 'normalequ':
            return model.normalequ(**_kwargs)

    def get_history(self):
        """
        Get the training history of current model
        :return: list as [iterations, [losses], [weights], [mis_class]]
        """
        if self.validation:
            return self.iterations, (self.losses, self.valid_losses), \
                   (self.weights), (self.misclass_rate, self.valid_misclass_rate)
        return self.iterations, self.losses, self.weights, self.misclass_rate

    def train(self, optimizer='sgd', loss_function='mse', **kwargs):
        """
        Train function to perform one time training
        Will based optimizer to select.
            TODO: Would add 'newton' in the future
        This
        :param optimizer: only support 'sgd'
        :param loss_function: loss_function name {mse, mae, logistic}
        :param kwargs: passed into sgd
        :return: best weight
        """
        self.loss_function = get_loss_function(loss_function)
        self.loss_function_name = loss_function

        if optimizer is 'sgd':
            self.sgd(**kwargs)

        return self.weights[-1]

    """===================================="""
    """ Beginning of the optimize Routines """
    """===================================="""

    def sgd(self, lr=0.01, decay=0.5, max_iters=1000,
            batch_size=128, early_stop=150, decay_intval=50, decay_lim=9):
        """
        Define the SGD algorithm here
            Implementing weight decay, early stop.

        :param lr:          learning rate
        :param decay:       weight decay after fix iterations
        :param max_iters:   maximum iterations
        :param batch_size:  batch_size
        :param early_stop:  early_stop after no improvement
        :return: final weight vector
        """
        np.set_printoptions(precision=4)
        w = self.weights[0]
        loss = self.compute_loss(self.train_y, self.train_x, w)
        best_loss = loss
        best_counter = 0
        decay_counter = 0
        # print("initial loss is {} ".format(loss))
        for epoch in range(max_iters):

            for batch_y, batch_x in batch_iter(self.train_y, self.train_x, batch_size):
                grad = self.get_gradient(batch_y, batch_x, w)
                w = w - lr * grad
            loss = self.compute_loss(self.train_y, self.train_x, w)
            mis_class = self.compute_metrics(self.train_y, self.train_x, w)

            self.weights.append(w)
            self.losses.append(loss)
            self.misclass_rate.append(mis_class)
            if self.validation is True:
                valid_loss = self.compute_loss(self.valid_y, self.valid_x, w)
                valid_mis_class = self.compute_metrics(self.valid_y, self.valid_x, w)
                self.valid_losses.append(valid_loss)
                self.valid_misclass_rate.append(valid_mis_class)
            # Display every 25 epoch
            if (epoch + 1) % 25 == 0:
                print('Epoch {e} in {m}'.format(e=epoch + 1, m=max_iters), end="\t")
                if self.validation is True:
                    # print('\tTrain Loss {0:0.4f}, \tTrain mis-class {0:0.4f}, '
                    #       '\tvalid loss {0:0.4f}, \tvalid mis-class {0:0.4f}'.
                    #       format(loss, mis_class, valid_loss, valid_mis_class))
                    print('\tTrain Loss {}, \tTrain mis-class {}, '
                          '\tvalid loss {}, \tvalid mis-class {}'.
                          format(loss, mis_class, valid_loss, valid_mis_class))
                else:
                    print('\tTrain Loss {}, \tTrain mis-class {}'.
                          format(loss, mis_class))
            # judge the performance
            if best_loss - loss > 0.000001:
                best_loss = loss
                best_counter = 0
            else:
                best_counter += 1
                if best_counter > early_stop:
                    print("Learning early stop since loss not improving for {} epoch.".format(best_counter))
                    break
                if best_counter % decay_intval == 0:
                    print("weight decay by {}".format(decay))
                    lr *= decay
                    decay_counter += 1
                    if decay_counter > decay_lim:
                        print("decay {} times, stop".format(decay_lim))
                        break
        return self.weights[-1]

    def newton(self, lr=0.01, max_iters=100):
        # TODO: implement newton method later
        raise NotImplementedError

    def cross_validation(self, cv, lambdas, lambda_name, seed=1, skip=False, plot=False, **kwargs):
        """
        Cross validation method to acquire the best prediction parameters.
        It will use the train_x y as data and do K-fold cross validation.

        :param cv:              cross validation times
        :param lambdas:         array of lambdas to be validated
        :param lambda_name:     the lambda name tag
        :param seed:            random seed
        :param skip:            skip the cross validation, only valid 1 time
        :param plot             plot cross-validation plot, if machine does not
                                support matplotlib.pyplot, set to false.
        :param kwargs:          other parameters could pass into compute_weight
        :return: best weights, best_lambda, (training error, valid error)
        """
        np.set_printoptions(precision=4)
        k_indices = build_k_indices(self.train_y, cv, seed)
        # define lists to store the loss of training data and test data
        err_tr = []
        err_te = []
        weights = []
        print("K-fold ({}) cross validation to examine [{}]".
              format(cv, lambdas))
        for lamb in lambdas:
            print("For lambda: {}".format(lamb))
            _mse_tr = []
            _mse_te = []
            _weight = []
            for k in range(cv):
                print('Cross valid iteration {}'.format(k))
                weight, loss_tr, loss_te = self._loop_cross_validation(self.train_y, self.train_x,
                                                                       k_indices, k,
                                                                       lamb, lambda_name, **kwargs)
                _mse_tr += [loss_tr]
                _mse_te += [loss_te]
                _weight.append(weight)
                if skip:
                    break
            avg_tr = np.average(_mse_tr)
            avg_te = np.average(_mse_te)
            err_tr += [avg_tr]
            err_te += [avg_te]
            weights.append(_weight)
            print("\t train error {}, \t valid error {}".
                  format(avg_tr, avg_te))
        # Select the best parameter during the cross validations.
        print('K-fold cross validation result: \n {} \n {}'.
              format(err_tr, err_te))
        # Select the best based on least err_te
        min_err_te = np.argmin(err_te)
        print('Best err_te result {}, lambda {}'.
              format(err_te[min_err_te], lambdas[min_err_te]))
        if plot:
            from plots import cross_validation_visualization
            cross_validation_visualization(lambdas, err_tr, err_te, title=lambda_name,
                                           error_name=self.loss_function_name)
        else:
            save_numpy_array(lambdas, err_tr, err_te, names=['lambda', 'err_tr', 'err_te'], title=self.regularizer.name)

        return weights[min_err_te], lambdas[min_err_te], (err_tr, err_te)

    def _loop_cross_validation(self, y, x, k_indices, k, lamb, lambda_name, **kwargs):
        """
        Single loop of cross validation
        :param y:           train labels
        :param x:           train data
        :param k_indices:   indices array
        :param k:           number of cross validations
        :param lamb:        lambda to use
        :param lambda_name: lambda_name to pass into compute weight
        :return:            weight, mis_tr, mis_te
        """
        train_ind = np.concatenate((k_indices[:k], k_indices[k + 1:]), axis=0)
        train_ind = np.reshape(train_ind, (train_ind.size,))
        test_ind = k_indices[k]

        # Note: different from np.ndarray, tuple is name[index,]
        # ndarray is name[index,:]
        train_x = x[train_ind,]
        train_y = y[train_ind,]
        test_x = x[test_ind,]
        test_y = y[test_ind,]
        # Insert one more kwargs item
        kwargs[lambda_name] = lamb

        weight = self.compute_weight(train_y, train_x, test_x, test_y, **kwargs)

        # Compute the metrics and return
        loss_tr = self.compute_metrics(train_y, train_x, weight)
        loss_te = self.compute_metrics(test_y, test_x, weight)

        return weight, loss_tr, loss_te

    def compute_metrics(self, target, data, weight):
        """
        Compute the following metrics
                Misclassification rate
        """
        pred = self.predict(data, weight)
        assert len(pred) == len(target)
        # Calculate the mis-classification rate:
        N = len(pred)
        pred = np.reshape(pred, (N,))
        target = np.reshape(target, (N,))
        nb_misclass = np.count_nonzero(target - pred)
        return nb_misclass / N

    def compute_loss(self, y, x, weight):
        return self.loss_function(y, x, weight)

class LogisticRegression(Model):
    """ Logistic regression """

    def __init__(self, train, validation=None, initial_weight=None,
                 loss_function_name='logistic',
                 calculate_weight='gradient',
                 regularizer=None, regularizer_p=None):
        """
        Constructor of Logistic Regression model
        :param train:           tuple (y, x)
        :param validation:      tuple (y, x)
        :param initial_weight:  weight vector, dim align x
        :param loss_function:   f(x, y, weight)
        :param regularizer:     "Ridge" || "Lasso"
        :param regularizer_p:   parameter
        """
        # Initialize the super class with given data.
        # Transform the y into {0,1}
        y, tx = train
        y[np.where(y < 0)] = 0
        train = (y, tx)
        if validation:
            val_y, val_tx = validation
            val_y[np.where(val_y < 0)] = 0
            validation = (val_y, val_tx)
        super(LogisticRegression, self).__init__(train, validation,
                                                 initial_weight=initial_weight,
                                                 loss_function_name=loss_function_name,
                                                 cal_weight=calculate_weight,
                                                 regularizer=regularizer,
                                                 regularizer_p=regularizer_p)
        # Set predicted label
        self.pred_label = [-1, 1]

    def __call__(self, x, weight=None):
        """
        Define the fit function and get prediction,
        generate probability of occurrence
        """
        if weight is None:
            weight = self.weights[-1]
        return sigmoid(np.dot(x, weight))

    def get_gradient(self, y, x, weight):
        """ calculate gradient given data and weight """
        y = np.reshape(y, (len(y),))
        return np.dot(x.T, sigmoid(np.dot(x, weight)) - y) \
               + self.regularizer.get_gradient(weight)

    def get_hessian(self, y, x, weight):
        # TODO: implement hessian for newton method
        raise NotImplementedError

    def predict(self, x, weight=None, cutting=0.5):
        """ Prediction of event {0,1} """
        if weight is None: weight = self.weights[-1]
        pred = sigmoid(np.dot(x, weight))
        pred[np.where(pred <= cutting)] = 0
        pred[np.where(pred > cutting)] = 1
        return pred

    def predict_label(self, x, weight=None, cutting=0.5, predict_label=None):
        """ Prediction result with labels """
        if predict_label is None:
            predict_label = self.pred_label
        if weight is None: weight = self.weights[-1]
        pred = self.predict(x, weight, cutting)
        pred[np.where(pred == 0)] = predict_label[0]
        pred[np.where(pred == 1)] = predict_label[1]
        return pred

    def train(self, loss_function='logistic',
              lr=0.1, decay=0.5, max_iters=3000, batch_size=128, **kwargs):
        """ Make the default loss logistic, set default parameters """
        return super(LogisticRegression, self).train('sgd', loss_function,
                                                     lr=lr,
                                                     decay=decay, max_iters=max_iters,
                                                     batch_size=batch_size, **kwargs)

    def normalequ(self, **kwargs):
        """ Should never call """
        raise NotImplementedError

class LinearRegression(Model):
    """ Linear regression model
        This is not fully tested, especially the cross-validation, please refers
        to the implemenations.py for linear model.
    """

    def __init__(self, train, validation=None, initial_weight=None,
                 regularizer=None, regularizer_p=None,
                 loss_function_name='mse', calculate_weight='normalequ'):
        # Initialize the super class with given data.
        super(LinearRegression, self).__init__(train, validation,
                                               initial_weight=initial_weight,
                                               loss_function_name=loss_function_name,
                                               cal_weight=calculate_weight,
                                               regularizer=regularizer,
                                               regularizer_p=regularizer_p)

    def __call__(self, x):
        """ calulate prediction based on latest result """
        return np.dot(x, self.weights[-1])

    def get_gradient(self, batch_y, batch_x, weight):
        """ return gradient of linear model, including the regularizer """
        N = batch_y.shape[0]
        grad = np.empty(len(weight))
        for index in range(N):
            _y = batch_y[index]
            _x = batch_x[index]
            grad = grad + gradient_least_square(_y, _x, weight, self.loss_function_name)
        grad /= N
        grad += self.regularizer.get_gradient(weight)
        return grad

    def predict(self, x, weight):
        """ Prediction function, predicting final result """
        pred = np.dot(x, weight)
        pred[np.where(pred <= 0)] = -1
        pred[np.where(pred > 0)] = 1
        return pred

    def normalequ(self):
        """ Normal equation to get parameters """
        tx = self.train_x
        y = self.train_y
        if self.regularizer is None:
            return np.linalg.solve(np.dot(tx.T, tx), np.dot(tx.T, y))
        elif self.regularizer.name is 'Ridge':
            G = np.eye(tx.shape[1])
            G[0, 0] = 0
            hes = np.dot(tx.T, tx) + self.regularizer_p * G
            return np.linalg.solve(hes, np.dot(tx.T, y))
        else:
            raise NotImplementedError


"""Base classes for all estimators."""
# Author: Gael Varoquaux <gael.varoquaux@normalesup.org>
# License: BSD 3 clause

import copy
import warnings

import numpy as np
from scipy import sparse
from .externals import six
from .utils.fixes import signature
from .utils.deprecation import deprecated
from .exceptions import ChangedBehaviorWarning as ChangedBehaviorWarning_

class ChangedBehaviorWarning(ChangedBehaviorWarning_):
    pass

ChangedBehaviorWarning = deprecated("ChangedBehaviorWarning has been moved "
                                    "into the sklearn.exceptions module. "
                                    "It will not be available here from "
                                    "version 0.19")(ChangedBehaviorWarning)

##############################################################################
def clone(estimator, safe=True):
    """Constructs a new estimator with the same parameters.

    Clone does a deep copy of the model in an estimator
    without actually copying attached data. It yields a new estimator
    with the same parameters that has not been fit on any data.

    Parameters
    ----------
    estimator: estimator object, or list, tuple or set of objects
        The estimator or group of estimators to be cloned

    safe: boolean, optional
        If safe is false, clone will fall back to a deepcopy on objects
        that are not estimators.

    """
    estimator_type = type(estimator)
    # XXX: not handling dictionaries
    if estimator_type in (list, tuple, set, frozenset):
        return estimator_type([clone(e, safe=safe) for e in estimator])
    elif not hasattr(estimator, 'get_params'):
        if not safe:
            return copy.deepcopy(estimator)
        else:
            raise TypeError("Cannot clone object '%s' (type %s): "
                            "it does not seem to be a scikit-learn estimator "
                            "as it does not implement a 'get_params' methods."
                            % (repr(estimator), type(estimator)))
    klass = estimator.__class__
    new_object_params = estimator.get_params(deep=False)
    for name, param in six.iteritems(new_object_params):
        new_object_params[name] = clone(param, safe=False)
    new_object = klass(**new_object_params)
    params_set = new_object.get_params(deep=False)

    # quick sanity check of the parameters of the clone
    for name in new_object_params:
        param1 = new_object_params[name]
        param2 = params_set[name]
        if isinstance(param1, np.ndarray):
            # For most ndarrays, we do not test for complete equality
            if not isinstance(param2, type(param1)):
                equality_test = False
            elif (param1.ndim > 0
                    and param1.shape[0] > 0
                    and isinstance(param2, np.ndarray)
                    and param2.ndim > 0
                    and param2.shape[0] > 0):
                equality_test = (
                    param1.shape == param2.shape
                    and param1.dtype == param2.dtype
                    # We have to use '.flat' for 2D arrays
                    and param1.flat[0] == param2.flat[0]
                    and param1.flat[-1] == param2.flat[-1]
                )
            else:
                equality_test = np.all(param1 == param2)
        elif sparse.issparse(param1):
            # For sparse matrices equality doesn't work
            if not sparse.issparse(param2):
                equality_test = False
            elif param1.size == 0 or param2.size == 0:
                equality_test = (
                    param1.__class__ == param2.__class__
                    and param1.size == 0
                    and param2.size == 0
                )
            else:
                equality_test = (
                    param1.__class__ == param2.__class__
                    and param1.data[0] == param2.data[0]
                    and param1.data[-1] == param2.data[-1]
                    and param1.nnz == param2.nnz
                    and param1.shape == param2.shape
                )
        else:
            new_obj_val = new_object_params[name]
            params_set_val = params_set[name]
            # The following construct is required to check equality on special
            # singletons such as np.nan that are not equal to them-selves:
            equality_test = (new_obj_val == params_set_val or
                             new_obj_val is params_set_val)
        if not equality_test:
            raise RuntimeError('Cannot clone object %s, as the constructor '
                               'does not seem to set parameter %s' %
                               (estimator, name))

    return new_object

###############################################################################
def _pprint(params, offset=0, printer=repr):
    """Pretty print the dictionary 'params'

    Parameters
    ----------
    params: dict
        The dictionary to pretty print

    offset: int
        The offset in characters to add at the begin of each line.

    printer:
        The function to convert entries to strings, typically
        the builtin str or repr

    """
    # Do a multi-line justified repr:
    options = np.get_printoptions()
    np.set_printoptions(precision=5, threshold=64, edgeitems=2)
    params_list = list()
    this_line_length = offset
    line_sep = ',\n' + (1 + offset // 2) * ' '
    for i, (k, v) in enumerate(sorted(six.iteritems(params))):
        if type(v) is float:
            # use str for representing floating point numbers
            # this way we get consistent representation across
            # architectures and versions.
            this_repr = '%s=%s' % (k, str(v))
        else:
            # use repr of the rest
            this_repr = '%s=%s' % (k, printer(v))
        if len(this_repr) > 500:
            this_repr = this_repr[:300] + '...' + this_repr[-100:]
        if i > 0:
            if (this_line_length + len(this_repr) >= 75 or '\n' in this_repr):
                params_list.append(line_sep)
                this_line_length = len(line_sep)
            else:
                params_list.append(', ')
                this_line_length += 2
        params_list.append(this_repr)
        this_line_length += len(this_repr)

    np.set_printoptions(**options)
    lines = ''.join(params_list)
    # Strip trailing space to avoid nightmare in doctests
    lines = '\n'.join(l.rstrip(' ') for l in lines.split('\n'))
    return lines

###############################################################################
class BaseEstimator(object):
    """Base class for all estimators in scikit-learn

    Notes
    -----
    All estimators should specify all the parameters that can be set
    at the class level in their ``__init__`` as explicit keyword
    arguments (no ``*args`` or ``**kwargs``).
    """

    @classmethod
    def _get_param_names(cls):
        """Get parameter names for the estimator"""
        # fetch the constructor or the original constructor before
        # deprecation wrapping if any
        init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
        if init is object.__init__:
            # No explicit constructor to introspect
            return []

        # introspect the constructor arguments to find the model parameters
        # to represent
        init_signature = signature(init)
        # Consider the constructor parameters excluding 'self'
        parameters = [p for p in init_signature.parameters.values()
                      if p.name != 'self' and p.kind != p.VAR_KEYWORD]
        for p in parameters:
            if p.kind == p.VAR_POSITIONAL:
                raise RuntimeError("scikit-learn estimators should always "
                                   "specify their parameters in the signature"
                                   " of their __init__ (no varargs)."
                                   " %s with constructor %s doesn't "
                                   " follow this convention."
                                   % (cls, init_signature))
        # Extract and sort argument names excluding 'self'
        return sorted([p.name for p in parameters])

    def get_params(self, deep=True):
        """Get parameters for this estimator.

        Parameters
        ----------
        deep: boolean, optional
            If True, will return the parameters for this estimator and
            contained subobjects that are estimators.

        Returns
        -------
        params : mapping of string to any
            Parameter names mapped to their values.
        """
        out = dict()
        for key in self._get_param_names():
            # We need deprecation warnings to always be on in order to
            # catch deprecated param values.
            # This is set in utils/__init__.py but it gets overwritten
            # when running under python3 somehow.
            warnings.simplefilter("always", DeprecationWarning)
            try:
                with warnings.catch_warnings(record=True) as w:
                    value = getattr(self, key, None)
                if len(w) and w[0].category == DeprecationWarning:
                    # if the parameter is deprecated, don't show it
                    continue
            finally:
                warnings.filters.pop(0)

            # XXX: should we rather test if instance of estimator?
            if deep and hasattr(value, 'get_params'):
                deep_items = value.get_params().items()
                out.update((key + '__' + k, val) for k, val in deep_items)
            out[key] = value
        return out

    def set_params(self, **params):
        """Set the parameters of this estimator.

        The method works on simple estimators as well as on nested objects
        (such as pipelines). The former have parameters of the form
        ``<component>__<parameter>`` so that it's possible to update each
        component of a nested object.

        Returns
        -------
        self
        """
        if not params:
            # Simple optimisation to gain speed (inspect is slow)
            return self
        valid_params = self.get_params(deep=True)
        for key, value in six.iteritems(params):
            split = key.split('__', 1)
            if len(split) > 1:
                # nested objects case
                name, sub_name = split
                if name not in valid_params:
                    raise ValueError('Invalid parameter %s for estimator %s. '
                                     'Check the list of available parameters '
                                     'with `estimator.get_params().keys()`.' %
                                     (name, self))
                sub_object = valid_params[name]
                sub_object.set_params(**{sub_name: value})
            else:
                # simple objects case
                if key not in valid_params:
                    raise ValueError('Invalid parameter %s for estimator %s. '
                                     'Check the list of available parameters '
                                     'with `estimator.get_params().keys()`.' %
                                     (key, self.__class__.__name__))
                setattr(self, key, value)
        return self

    def __repr__(self):
        class_name = self.__class__.__name__
        return '%s(%s)' % (class_name, _pprint(self.get_params(deep=False),
                                               offset=len(class_name),),)

###############################################################################
class ClassifierMixin(object):
    """Mixin class for all classifiers in scikit-learn."""
    _estimator_type = "classifier"

    def score(self, X, y, sample_weight=None):
        """Returns the mean accuracy on the given test data and labels.

        In multi-label classification, this is the subset accuracy
        which is a harsh metric since you require for each sample that
        each label set be correctly predicted.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Test samples.

        y : array-like, shape = (n_samples) or (n_samples, n_outputs)
            True labels for X.

        sample_weight : array-like, shape = [n_samples], optional
            Sample weights.

        Returns
        -------
        score : float
            Mean accuracy of self.predict(X) wrt. y.

        """
        from .metrics import accuracy_score
        return accuracy_score(y, self.predict(X), sample_weight=sample_weight)

###############################################################################
class RegressorMixin(object):
    """Mixin class for all regression estimators in scikit-learn."""
    _estimator_type = "regressor"

    def score(self, X, y, sample_weight=None):
        """Returns the coefficient of determination R^2 of the prediction.

        The coefficient R^2 is defined as (1 - u/v), where u is the regression
        sum of squares ((y_true - y_pred) ** 2).sum() and v is the residual
        sum of squares ((y_true - y_true.mean()) ** 2).sum().
        Best possible score is 1.0 and it can be negative (because the
        model can be arbitrarily worse). A constant model that always
        predicts the expected value of y, disregarding the input features,
        would get a R^2 score of 0.0.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Test samples.

        y : array-like, shape = (n_samples) or (n_samples, n_outputs)
            True values for X.

        sample_weight : array-like, shape = [n_samples], optional
            Sample weights.

        Returns
        -------
        score : float
            R^2 of self.predict(X) wrt. y.
        """

        from .metrics import r2_score
        return r2_score(y, self.predict(X), sample_weight=sample_weight,
                        multioutput='variance_weighted')

###############################################################################
class ClusterMixin(object):
    """Mixin class for all cluster estimators in scikit-learn."""
    _estimator_type = "clusterer"

    def fit_predict(self, X, y=None):
        """Performs clustering on X and returns cluster labels.

        Parameters
        ----------
        X : ndarray, shape (n_samples, n_features)
            Input data.

        Returns
        -------
        y : ndarray, shape (n_samples,)
            cluster labels
        """
        # non-optimized default implementation; override when a better
        # method is possible for a given clustering algorithm
        self.fit(X)
        return self.labels_

class BiclusterMixin(object):
    """Mixin class for all bicluster estimators in scikit-learn"""

    @property
    def biclusters_(self):
        """Convenient way to get row and column indicators together.

        Returns the ``rows_`` and ``columns_`` members.
        """
        return self.rows_, self.columns_

    def get_indices(self, i):
        """Row and column indices of the i'th bicluster.

        Only works if ``rows_`` and ``columns_`` attributes exist.

        Returns
        -------
        row_ind : np.array, dtype=np.intp
            Indices of rows in the dataset that belong to the bicluster.
        col_ind : np.array, dtype=np.intp
            Indices of columns in the dataset that belong to the bicluster.

        """
        rows = self.rows_[i]
        columns = self.columns_[i]
        return np.nonzero(rows)[0], np.nonzero(columns)[0]

    def get_shape(self, i):
        """Shape of the i'th bicluster.

        Returns
        -------
        shape : (int, int)
            Number of rows and columns (resp.) in the bicluster.
        """
        indices = self.get_indices(i)
        return tuple(len(i) for i in indices)

    def get_submatrix(self, i, data):
        """Returns the submatrix corresponding to bicluster `i`.

        Works with sparse matrices. Only works if ``rows_`` and
        ``columns_`` attributes exist.

        """
        from .utils.validation import check_array
        data = check_array(data, accept_sparse='csr')
        row_ind, col_ind = self.get_indices(i)
        return data[row_ind[:, np.newaxis], col_ind]

###############################################################################
class TransformerMixin(object):
    """Mixin class for all transformers in scikit-learn."""

    def fit_transform(self, X, y=None, **fit_params):
        """Fit to data, then transform it.

        Fits transformer to X and y with optional parameters fit_params
        and returns a transformed version of X.

        Parameters
        ----------
        X : numpy array of shape [n_samples, n_features]
            Training set.

        y : numpy array of shape [n_samples]
            Target values.

        Returns
        -------
        X_new : numpy array of shape [n_samples, n_features_new]
            Transformed array.

        """
        # non-optimized default implementation; override when a better
        # method is possible for a given clustering algorithm
        if y is None:
            # fit method of arity 1 (unsupervised transformation)
            return self.fit(X, **fit_params).transform(X)
        else:
            # fit method of arity 2 (supervised transformation)
            return self.fit(X, y, **fit_params).transform(X)

###############################################################################
class MetaEstimatorMixin(object):
    """Mixin class for all meta estimators in scikit-learn."""
    # this is just a tag for the moment

###############################################################################

def is_classifier(estimator):
    """Returns True if the given estimator is (probably) a classifier."""
    return getattr(estimator, "_estimator_type", None) == "classifier"

def is_regressor(estimator):
    """Returns True if the given estimator is (probably) a regressor."""
    return getattr(estimator, "_estimator_type", None) == "regressor"


from __future__ import absolute_import
import pandas as pd
from os.path import join
import sys
from . import _data_dir

'''
This module provides a pandas DataFrame instance of four
of the datasets from gapminder.org.

These are read in from csvs that have been downloaded from Bokeh's
sample data on S3. But the original code that generated the csvs from the
raw gapminder data is available at the bottom of this file.
'''

data_dir = _data_dir()

datasets = [
    'fertility',
    'life_expectancy',
    'population',
    'regions',
]

for dataset in datasets:
    filename = join(data_dir, 'gapminder_%s.csv' % dataset)
    try:
        setattr(
            sys.modules[__name__],
            dataset,
            pd.read_csv(filename, index_col='Country')
        )
    except (IOError, OSError):
        raise RuntimeError('Could not load gapminder data file "%s". Please execute bokeh.sampledata.download()' % filename)

__all__ = datasets

# ====================================================

# Original data is from Gapminder - www.gapminder.org.
# The google docs links are maintained by gapminder

# The following script was used to get the data from gapminder
# and process it into the csvs stored in bokeh's sampledata.

"""
population_url = "http://spreadsheets.google.com/pub?key=phAwcNAVuyj0XOoBL_n5tAQ&output=xls"
fertility_url = "http://spreadsheets.google.com/pub?key=phAwcNAVuyj0TAlJeCEzcGQ&output=xls"
life_expectancy_url = "http://spreadsheets.google.com/pub?key=tiAiXcrneZrUnnJ9dBU-PAw&output=xls"
regions_url = "https://docs.google.com/spreadsheets/d/1OxmGUNWeADbPJkQxVPupSOK5MbAECdqThnvyPrwG5Os/pub?gid=1&output=xls"

def _get_data(url):
    # Get the data from the url and return only 1962 - 2013
    df = pd.read_excel(url, index_col=0)
    df = df.unstack().unstack()
    df = df[(df.index >= 1964) & (df.index <= 2013)]
    df = df.unstack().unstack()
    return df

fertility_df = _get_data(fertility_url)
life_expectancy_df = _get_data(life_expectancy_url)
population_df = _get_data(population_url)
regions_df = pd.read_excel(regions_url, index_col=0)

# have common countries across all data
fertility_df = fertility_df.drop(fertility_df.index.difference(life_expectancy_df.index))
population_df = population_df.drop(population_df.index.difference(life_expectancy_df.index))
regions_df = regions_df.drop(regions_df.index.difference(life_expectancy_df.index))

fertility_df.to_csv('gapminder_fertility.csv')
population_df.to_csv('gapminder_population.csv')
life_expectancy_df.to_csv('gapminder_life_expectancy.csv')
regions_df.to_csv('gapminder_regions.csv')
"""

# ======================================================