#
# Copyright 2015 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import random
import numpy as np
import pandas as pd

from zipline.finance.trading import TradingEnvironment
from zipline.data.us_equity_minutes import BcolzMinuteBarWriter

def generate_daily_test_data(first_day,
                             last_day,
                             starting_open,
                             starting_volume,
                             multipliers_list,
                             path):

    days = TradingEnvironment.instance().days_in_range(first_day, last_day)

    days_count = len(days)
    o = np.zeros(days_count, dtype=np.uint32)
    h = np.zeros(days_count, dtype=np.uint32)
    l = np.zeros(days_count, dtype=np.uint32)
    c = np.zeros(days_count, dtype=np.uint32)
    v = np.zeros(days_count, dtype=np.uint32)

    last_open = starting_open * 1000
    last_volume = starting_volume

    for idx in range(days_count):
        new_open = last_open + round((random.random() * 5), 2)

        o[idx] = new_open
        h[idx] = new_open + round((random.random() * 10000), 2)
        l[idx] = new_open - round((random.random() * 10000),  2)
        c[idx] = (h[idx] + l[idx]) / 2
        v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4))

        last_open = o[idx]
        last_volume = v[idx]

    # now deal with multipliers
    if len(multipliers_list) > 0:
        range_start = 0

        for multiplier_info in multipliers_list:
            range_end = days.searchsorted(multiplier_info[0])

            # dividing by the multiplier because we're going backwards
            # and generating the original data that will then be adjusted.
            o[range_start:range_end] /= multiplier_info[1]
            h[range_start:range_end] /= multiplier_info[1]
            l[range_start:range_end] /= multiplier_info[1]
            c[range_start:range_end] /= multiplier_info[1]
            v[range_start:range_end] *= multiplier_info[1]

            range_start = range_end

    df = pd.DataFrame({
        "open": o,
        "high": h,
        "low": l,
        "close": c,
        "volume": v
    }, columns=[
        "open",
        "high",
        "low",
        "close",
        "volume"
    ], index=days)

    df.to_csv(path, index_label="day")

def generate_minute_test_data(first_day,
                              last_day,
                              starting_open,
                              starting_volume,
                              multipliers_list,
                              path):
    """
    Utility method to generate fake minute-level CSV data.
    :param first_day: first trading day
    :param last_day: last trading day
    :param starting_open: first open value, raw value.
    :param starting_volume: first volume value, raw value.
    :param multipliers_list: ordered list of pd.Timestamp -> float, one per day
            in the range
    :param path: path to save the CSV
    :return: None
    """

    full_minutes = BcolzMinuteBarWriter.full_minutes_for_days(
        first_day, last_day)
    minutes_count = len(full_minutes)

    minutes = TradingEnvironment.instance().minutes_for_days_in_range(
        first_day, last_day)

    o = np.zeros(minutes_count, dtype=np.uint32)
    h = np.zeros(minutes_count, dtype=np.uint32)
    l = np.zeros(minutes_count, dtype=np.uint32)
    c = np.zeros(minutes_count, dtype=np.uint32)
    v = np.zeros(minutes_count, dtype=np.uint32)

    last_open = starting_open * 1000
    last_volume = starting_volume

    for minute in minutes:
        # ugly, but works
        idx = full_minutes.searchsorted(minute)

        new_open = last_open + round((random.random() * 5), 2)

        o[idx] = new_open
        h[idx] = new_open + round((random.random() * 10000), 2)
        l[idx] = new_open - round((random.random() * 10000),  2)
        c[idx] = (h[idx] + l[idx]) / 2
        v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4))

        last_open = o[idx]
        last_volume = v[idx]

    # now deal with multipliers
    if len(multipliers_list) > 0:
        for idx, multiplier_info in enumerate(multipliers_list):
            start_idx = idx * 390
            end_idx = start_idx + 390

            # dividing by the multipler because we're going backwards
            # and generating the original data that will then be adjusted.
            o[start_idx:end_idx] /= multiplier_info[1]
            h[start_idx:end_idx] /= multiplier_info[1]
            l[start_idx:end_idx] /= multiplier_info[1]
            c[start_idx:end_idx] /= multiplier_info[1]
            v[start_idx:end_idx] *= multiplier_info[1]

    df = pd.DataFrame({
        "open": o,
        "high": h,
        "low": l,
        "close": c,
        "volume": v
    }, columns=[
        "open",
        "high",
        "low",
        "close",
        "volume"
    ], index=minutes)

    df.to_csv(path, index_label="minute")


"""
============================================================
Parameter estimation using grid search with cross-validation
============================================================

This examples shows how a classifier is optimized by cross-validation,
which is done using the :class:`sklearn.grid_search.GridSearchCV` object
on a development set that comprises only half of the available labeled data.

The performance of the selected hyper-parameters and trained model is
then measured on a dedicated evaluation set that was not used during
the model selection step.

More details on tools available for model selection can be found in the
sections on :ref:`cross_validation` and :ref:`grid_search`.

"""

from __future__ import print_function

from sklearn import datasets
from sklearn.cross_validation import train_test_split
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.svm import SVC

print(__doc__)

# Loading the Digits dataset
digits = datasets.load_digits()

# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.images)
X = digits.images.reshape((n_samples, -1))
y = digits.target

# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.5, random_state=0)

# Set the parameters by cross-validation
tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 10, 100, 1000]},
                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]

scores = ['precision', 'recall']

for score in scores:
    print("# Tuning hyper-parameters for %s" % score)
    print()

    clf = GridSearchCV(SVC(C=1), tuned_parameters, cv=5,
                       scoring='%s_weighted' % score)
    clf.fit(X_train, y_train)

    print("Best parameters set found on development set:")
    print()
    print(clf.best_params_)
    print()
    print("Grid scores on development set:")
    print()
    for params, mean_score, scores in clf.grid_scores_:
        print("%0.3f (+/-%0.03f) for %r"
              % (mean_score, scores.std() * 2, params))
    print()

    print("Detailed classification report:")
    print()
    print("The model is trained on the full development set.")
    print("The scores are computed on the full evaluation set.")
    print()
    y_true, y_pred = y_test, clf.predict(X_test)
    print(classification_report(y_true, y_pred))
    print()

# Note the problem is too easy: the hyperparameter plateau is too flat and the
# output model is the same for precision and recall with ties in quality.


#!/usr/bin/env python
"""
    rawdata_se_objects_split.py

    Example:

    rawdata_se_objects_split.py --help

    rawdata_se_objects_split.py --inputFileListFile inputfilelist.csv
                                --outputFileListFile outputfilelist.csv
                                --verbose 2

    """

##################################

def main():

    import os
    import argparse
    import time

    """Create command line arguments"""
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--inputFileListFile', help='name of CSV file containing list of input files', default='inputfilelist.csv')
    parser.add_argument('--outputFileListFile', help='name of CSV file containing list of filter bands and output files', default='outputfilelist.csv')
    parser.add_argument('--verbose', help='verbosity level of output to screen (0,1,2,...)', default=0, type=int)
    args = parser.parse_args()

    if args.verbose > 0: print args

    # Execute method...
    status = rawdata_se_objects_split(args)

##################################
# rawdata_se_objects_split
#
# Based on sepBands from y2a1_tertiaries.py
#

def rawdata_se_objects_split(args):

    import os
    import datetime
    import numpy as np
    import pandas as pd
    from astropy.table import Table

    if args.verbose>0:
        print
        print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
        print 'rawdata_se_objects_split'
        print '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
        print

    # Read in inputFileListFile...
    inputFileListFile = args.inputFileListFile
    if os.path.isfile(inputFileListFile)==False:
        print """Input filelist file %s does not exist...""" % (inputFileListFile)
        print """Exiting rawdata_se_objects_split method with return code 1"""
        return 1
    inputFileListDF = pd.read_csv(inputFileListFile,
                                  header=None,
                                  names=['FILENAME'],
                                  comment='#')
    inputFileListDF['FILENAME'] = inputFileListDF['FILENAME'].str.strip()
    inputFileListSeries = inputFileListDF['FILENAME']
    inputFileList = inputFileListSeries.values.tolist()

    if args.verbose>1:
        print 'Input file list:'
        print inputFileList

    # Read in outputFileListFile and convert it to a python
    #  dictionary, ensuring there are no extraneous white spaces
    #  in the file names listed...
    outputFileListFile = args.outputFileListFile
    if os.path.isfile(outputFileListFile)==False:
        print """Output filelist file %s does not exist...""" % (outputFileListFile)
        print """Exiting rawdata_se_objects_split method with return code 1"""
        return 1
    outputFileListDF = pd.read_csv(outputFileListFile,
                                  header=None,
                                  names=['BAND','FILENAME'],
                                  index_col='BAND',
                                  comment='#')
    outputFileListDF['FILENAME'] = outputFileListDF['FILENAME'].str.strip()
    outputFileListSeries = outputFileListDF['FILENAME']
    outputFileListDict = outputFileListSeries.to_dict()

    # Also, grab the band list from the outputFileListFile series...
    bandList = outputFileListSeries.index.values.tolist()

    if args.verbose>1:
        print 'Output file list dictionary:'
        print outputFileListDict
        print 'Band list:'
        print bandList

    # Loop through inputFileList...
    firstFile=True
    for inputFile in inputFileList:

        if args.verbose > 1:
            print """Working on input file %s...""" % inputFile
            print datetime.datetime.now()

        # Read in file...
        t = Table.read(inputFile)

        # Convert astropy Table to pandas dataframe...
        df = t.to_pandas()

        # Verify that 'BAND' is a column in the dataframe;
        #  otherwise, skip...
        if 'BAND' not in df.columns:
            print """Could not find 'BAND' in header of %s...  Skipping""" \
                % (inputFile)
            del df
            continue

        # Verify that 'FILENAME' is a column in the dataframe;
        #  otherwise, skip...
        if 'FILENAME' not in df.columns:
            print """Could not find 'FILENAME' in header of %s...  Skipping""" \
                % (inputFile)
            del df
            continue

        # Trim leading and trailing white space from the FILENAME column...
        df['FILENAME'] = df['FILENAME'].str.strip()

        # Trim leading and trailing white space from the BAND column...
        df['BAND'] = df['BAND'].str.strip()

        # If this is the first (valid) file, create initial
        #   output files (empty except for the CSV header)...
        if firstFile is True:
            for band in bandList:
                outputFile = outputFileListDict[band]
                # Create a mask with all entries equal to False...
                mask = pd.Series(np.zeros(df.BAND.size, dtype=bool))
                df[mask].to_csv(outputFile,index=False)
            firstFile = False

        # Loop through band list, appending the rows from
        #  each band to the appropriate output file...
        for band in bandList:
            outputFile = outputFileListDict[band]
            mask = (df.BAND == band)
            # Faster if we move the "open" to outside the loop?:
            with open(outputFile, 'a') as f:
                df[mask].to_csv(f, index=False, header=False)
            f.close()

        # Clean up some space before moving to next file...
        del df
        #del t

        if args.verbose > 1:
            print datetime.datetime.now()

    if args.verbose>0: print

    return 0

##################################

if __name__ == "__main__":
    main()

##################################


import pickle
import unittest

import numpy as np
import scipy.sparse as sp

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import raises
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_false, assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises_regexp

from sklearn import linear_model, datasets, metrics
from sklearn.base import clone
from sklearn.linear_model import SGDClassifier, SGDRegressor
from sklearn.preprocessing import LabelEncoder, scale, MinMaxScaler

class SparseSGDClassifier(SGDClassifier):

    def fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).fit(X, y, *args, **kw)

    def partial_fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).partial_fit(X, y, *args, **kw)

    def decision_function(self, X):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).decision_function(X)

    def predict_proba(self, X):
        X = sp.csr_matrix(X)
        return super(SparseSGDClassifier, self).predict_proba(X)

class SparseSGDRegressor(SGDRegressor):

    def fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return SGDRegressor.fit(self, X, y, *args, **kw)

    def partial_fit(self, X, y, *args, **kw):
        X = sp.csr_matrix(X)
        return SGDRegressor.partial_fit(self, X, y, *args, **kw)

    def decision_function(self, X, *args, **kw):
        X = sp.csr_matrix(X)
        return SGDRegressor.decision_function(self, X, *args, **kw)

# Test Data

# test sample 1
X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
Y = [1, 1, 1, 2, 2, 2]
T = np.array([[-1, -1], [2, 2], [3, 2]])
true_result = [1, 2, 2]

# test sample 2; string class labels
X2 = np.array([[-1, 1], [-0.75, 0.5], [-1.5, 1.5],
               [1, 1], [0.75, 0.5], [1.5, 1.5],
               [-1, -1], [0, -0.5], [1, -1]])
Y2 = ["one"] * 3 + ["two"] * 3 + ["three"] * 3
T2 = np.array([[-1.5, 0.5], [1, 2], [0, -2]])
true_result2 = ["one", "two", "three"]

# test sample 3
X3 = np.array([[1, 1, 0, 0, 0, 0], [1, 1, 0, 0, 0, 0],
               [0, 0, 1, 0, 0, 0], [0, 0, 1, 0, 0, 0],
               [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 1, 1],
               [0, 0, 0, 1, 0, 0], [0, 0, 0, 1, 0, 0]])
Y3 = np.array([1, 1, 1, 1, 2, 2, 2, 2])

# test sample 4 - two more or less redundent feature groups
X4 = np.array([[1, 0.9, 0.8, 0, 0, 0], [1, .84, .98, 0, 0, 0],
               [1, .96, .88, 0, 0, 0], [1, .91, .99, 0, 0, 0],
               [0, 0, 0, .89, .91, 1], [0, 0, 0, .79, .84, 1],
               [0, 0, 0, .91, .95, 1], [0, 0, 0, .93, 1, 1]])
Y4 = np.array([1, 1, 1, 1, 2, 2, 2, 2])

iris = datasets.load_iris()

# test sample 5 - test sample 1 as binary classification problem
X5 = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
Y5 = [1, 1, 1, 2, 2, 2]
true_result5 = [0, 1, 1]

# Classification Test Case

class CommonTest(object):

    def factory(self, **kwargs):
        if "random_state" not in kwargs:
            kwargs["random_state"] = 42
        return self.factory_class(**kwargs)

    # a simple implementation of ASGD to use for testing
    # uses squared loss to find the gradient
    def asgd(self, X, y, eta, alpha, weight_init=None, intercept_init=0.0):
        if weight_init is None:
            weights = np.zeros(X.shape[1])
        else:
            weights = weight_init

        average_weights = np.zeros(X.shape[1])
        intercept = intercept_init
        average_intercept = 0.0
        decay = 1.0

        # sparse data has a fixed decay of .01
        if (isinstance(self, SparseSGDClassifierTestCase) or
                isinstance(self, SparseSGDRegressorTestCase)):
            decay = .01

        for i, entry in enumerate(X):
            p = np.dot(entry, weights)
            p += intercept
            gradient = p - y[i]
            weights *= 1.0 - (eta * alpha)
            weights += -(eta * gradient * entry)
            intercept += -(eta * gradient) * decay

            average_weights *= i
            average_weights += weights
            average_weights /= i + 1.0

            average_intercept *= i
            average_intercept += intercept
            average_intercept /= i + 1.0

        return average_weights, average_intercept

    def _test_warm_start(self, X, Y, lr):
        # Test that explicit warm restart...
        clf = self.factory(alpha=0.01, eta0=0.01, n_iter=5, shuffle=False,
                           learning_rate=lr)
        clf.fit(X, Y)

        clf2 = self.factory(alpha=0.001, eta0=0.01, n_iter=5, shuffle=False,
                            learning_rate=lr)
        clf2.fit(X, Y,
                 coef_init=clf.coef_.copy(),
                 intercept_init=clf.intercept_.copy())

        # ... and implicit warm restart are equivalent.
        clf3 = self.factory(alpha=0.01, eta0=0.01, n_iter=5, shuffle=False,
                            warm_start=True, learning_rate=lr)
        clf3.fit(X, Y)

        assert_equal(clf3.t_, clf.t_)
        assert_array_almost_equal(clf3.coef_, clf.coef_)

        clf3.set_params(alpha=0.001)
        clf3.fit(X, Y)

        assert_equal(clf3.t_, clf2.t_)
        assert_array_almost_equal(clf3.coef_, clf2.coef_)

    def test_warm_start_constant(self):
        self._test_warm_start(X, Y, "constant")

    def test_warm_start_invscaling(self):
        self._test_warm_start(X, Y, "invscaling")

    def test_warm_start_optimal(self):
        self._test_warm_start(X, Y, "optimal")

    def test_input_format(self):
        # Input format tests.
        clf = self.factory(alpha=0.01, n_iter=5,
                           shuffle=False)
        clf.fit(X, Y)
        Y_ = np.array(Y)[:, np.newaxis]

        Y_ = np.c_[Y_, Y_]
        assert_raises(ValueError, clf.fit, X, Y_)

    def test_clone(self):
        # Test whether clone works ok.
        clf = self.factory(alpha=0.01, n_iter=5, penalty='l1')
        clf = clone(clf)
        clf.set_params(penalty='l2')
        clf.fit(X, Y)

        clf2 = self.factory(alpha=0.01, n_iter=5, penalty='l2')
        clf2.fit(X, Y)

        assert_array_equal(clf.coef_, clf2.coef_)

    def test_plain_has_no_average_attr(self):
        clf = self.factory(average=True, eta0=.01)
        clf.fit(X, Y)

        assert_true(hasattr(clf, 'average_coef_'))
        assert_true(hasattr(clf, 'average_intercept_'))
        assert_true(hasattr(clf, 'standard_intercept_'))
        assert_true(hasattr(clf, 'standard_coef_'))

        clf = self.factory()
        clf.fit(X, Y)

        assert_false(hasattr(clf, 'average_coef_'))
        assert_false(hasattr(clf, 'average_intercept_'))
        assert_false(hasattr(clf, 'standard_intercept_'))
        assert_false(hasattr(clf, 'standard_coef_'))

    def test_late_onset_averaging_not_reached(self):
        clf1 = self.factory(average=600)
        clf2 = self.factory()
        for _ in range(100):
            if isinstance(clf1, SGDClassifier):
                clf1.partial_fit(X, Y, classes=np.unique(Y))
                clf2.partial_fit(X, Y, classes=np.unique(Y))
            else:
                clf1.partial_fit(X, Y)
                clf2.partial_fit(X, Y)

        assert_array_almost_equal(clf1.coef_, clf2.coef_, decimal=16)
        assert_almost_equal(clf1.intercept_, clf2.intercept_, decimal=16)

    def test_late_onset_averaging_reached(self):
        eta0 = .001
        alpha = .0001
        Y_encode = np.array(Y)
        Y_encode[Y_encode == 1] = -1.0
        Y_encode[Y_encode == 2] = 1.0

        clf1 = self.factory(average=7, learning_rate="constant",
                            loss='squared_loss', eta0=eta0,
                            alpha=alpha, n_iter=2, shuffle=False)
        clf2 = self.factory(average=0, learning_rate="constant",
                            loss='squared_loss', eta0=eta0,
                            alpha=alpha, n_iter=1, shuffle=False)

        clf1.fit(X, Y_encode)
        clf2.fit(X, Y_encode)

        average_weights, average_intercept = \
            self.asgd(X, Y_encode, eta0, alpha,
                      weight_init=clf2.coef_.ravel(),
                      intercept_init=clf2.intercept_)

        assert_array_almost_equal(clf1.coef_.ravel(),
                                  average_weights.ravel(),
                                  decimal=16)
        assert_almost_equal(clf1.intercept_, average_intercept, decimal=16)

class DenseSGDClassifierTestCase(unittest.TestCase, CommonTest):
    """Test suite for the dense representation variant of SGD"""
    factory_class = SGDClassifier

    def test_sgd(self):
        # Check that SGD gives any results :-)

        for loss in ("hinge", "squared_hinge", "log", "modified_huber"):
            clf = self.factory(penalty='l2', alpha=0.01, fit_intercept=True,
                               loss=loss, n_iter=10, shuffle=True)
            clf.fit(X, Y)
            # assert_almost_equal(clf.coef_[0], clf.coef_[1], decimal=7)
            assert_array_equal(clf.predict(T), true_result)

    @raises(ValueError)
    def test_sgd_bad_l1_ratio(self):
        # Check whether expected ValueError on bad l1_ratio
        self.factory(l1_ratio=1.1)

    @raises(ValueError)
    def test_sgd_bad_learning_rate_schedule(self):
        # Check whether expected ValueError on bad learning_rate
        self.factory(learning_rate="<unknown>")

    @raises(ValueError)
    def test_sgd_bad_eta0(self):
        # Check whether expected ValueError on bad eta0
        self.factory(eta0=0, learning_rate="constant")

    @raises(ValueError)
    def test_sgd_bad_alpha(self):
        # Check whether expected ValueError on bad alpha
        self.factory(alpha=-.1)

    @raises(ValueError)
    def test_sgd_bad_penalty(self):
        # Check whether expected ValueError on bad penalty
        self.factory(penalty='foobar', l1_ratio=0.85)

    @raises(ValueError)
    def test_sgd_bad_loss(self):
        # Check whether expected ValueError on bad loss
        self.factory(loss="foobar")

    @raises(ValueError)
    def test_sgd_n_iter_param(self):
        # Test parameter validity check
        self.factory(n_iter=-10000)

    @raises(ValueError)
    def test_sgd_shuffle_param(self):
        # Test parameter validity check
        self.factory(shuffle="false")

    @raises(TypeError)
    def test_argument_coef(self):
        # Checks coef_init not allowed as model argument (only fit)
        # Provided coef_ does not match dataset.
        self.factory(coef_init=np.zeros((3,))).fit(X, Y)

    @raises(ValueError)
    def test_provide_coef(self):
        # Checks coef_init shape for the warm starts
        # Provided coef_ does not match dataset.
        self.factory().fit(X, Y, coef_init=np.zeros((3,)))

    @raises(ValueError)
    def test_set_intercept(self):
        # Checks intercept_ shape for the warm starts
        # Provided intercept_ does not match dataset.
        self.factory().fit(X, Y, intercept_init=np.zeros((3,)))

    def test_set_intercept_binary(self):
        # Checks intercept_ shape for the warm starts in binary case
        self.factory().fit(X5, Y5, intercept_init=0)

    def test_average_binary_computed_correctly(self):
        # Checks the SGDClassifier correctly computes the average weights
        eta = .1
        alpha = 2.
        n_samples = 20
        n_features = 10
        rng = np.random.RandomState(0)
        X = rng.normal(size=(n_samples, n_features))
        w = rng.normal(size=n_features)

        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        # simple linear function without noise
        y = np.dot(X, w)
        y = np.sign(y)

        clf.fit(X, y)

        average_weights, average_intercept = self.asgd(X, y, eta, alpha)
        average_weights = average_weights.reshape(1, -1)
        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=14)
        assert_almost_equal(clf.intercept_, average_intercept, decimal=14)

    def test_set_intercept_to_intercept(self):
        # Checks intercept_ shape consistency for the warm starts
        # Inconsistent intercept_ shape.
        clf = self.factory().fit(X5, Y5)
        self.factory().fit(X5, Y5, intercept_init=clf.intercept_)
        clf = self.factory().fit(X, Y)
        self.factory().fit(X, Y, intercept_init=clf.intercept_)

    @raises(ValueError)
    def test_sgd_at_least_two_labels(self):
        # Target must have at least two labels
        self.factory(alpha=0.01, n_iter=20).fit(X2, np.ones(9))

    def test_partial_fit_weight_class_balanced(self):
        # partial_fit with class_weight='balanced' not supported"""
        assert_raises_regexp(ValueError,
                             "class_weight 'balanced' is not supported for "
                             "partial_fit. In order to use 'balanced' weights, "
                             "use compute_class_weight\('balanced', classes, y\). "
                             "In place of y you can us a large enough sample "
                             "of the full training set target to properly "
                             "estimate the class frequency distributions. "
                             "Pass the resulting weights as the class_weight "
                             "parameter.",
                             self.factory(class_weight='balanced').partial_fit,
                             X, Y, classes=np.unique(Y))

    def test_sgd_multiclass(self):
        # Multi-class test case
        clf = self.factory(alpha=0.01, n_iter=20).fit(X2, Y2)
        assert_equal(clf.coef_.shape, (3, 2))
        assert_equal(clf.intercept_.shape, (3,))
        assert_equal(clf.decision_function([0, 0]).shape, (1, 3))
        pred = clf.predict(T2)
        assert_array_equal(pred, true_result2)

    def test_sgd_multiclass_average(self):
        eta = .001
        alpha = .01
        # Multi-class average test case
        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        np_Y2 = np.array(Y2)
        clf.fit(X2, np_Y2)
        classes = np.unique(np_Y2)

        for i, cl in enumerate(classes):
            y_i = np.ones(np_Y2.shape[0])
            y_i[np_Y2 != cl] = -1
            average_coef, average_intercept = self.asgd(X2, y_i, eta, alpha)
            assert_array_almost_equal(average_coef, clf.coef_[i], decimal=16)
            assert_almost_equal(average_intercept,
                                clf.intercept_[i],
                                decimal=16)

    def test_sgd_multiclass_with_init_coef(self):
        # Multi-class test case
        clf = self.factory(alpha=0.01, n_iter=20)
        clf.fit(X2, Y2, coef_init=np.zeros((3, 2)),
                intercept_init=np.zeros(3))
        assert_equal(clf.coef_.shape, (3, 2))
        assert_true(clf.intercept_.shape, (3,))
        pred = clf.predict(T2)
        assert_array_equal(pred, true_result2)

    def test_sgd_multiclass_njobs(self):
        # Multi-class test case with multi-core support
        clf = self.factory(alpha=0.01, n_iter=20, n_jobs=2).fit(X2, Y2)
        assert_equal(clf.coef_.shape, (3, 2))
        assert_equal(clf.intercept_.shape, (3,))
        assert_equal(clf.decision_function([0, 0]).shape, (1, 3))
        pred = clf.predict(T2)
        assert_array_equal(pred, true_result2)

    def test_set_coef_multiclass(self):
        # Checks coef_init and intercept_init shape for for multi-class
        # problems
        # Provided coef_ does not match dataset
        clf = self.factory()
        assert_raises(ValueError, clf.fit, X2, Y2, coef_init=np.zeros((2, 2)))

        # Provided coef_ does match dataset
        clf = self.factory().fit(X2, Y2, coef_init=np.zeros((3, 2)))

        # Provided intercept_ does not match dataset
        clf = self.factory()
        assert_raises(ValueError, clf.fit, X2, Y2,
                      intercept_init=np.zeros((1,)))

        # Provided intercept_ does match dataset.
        clf = self.factory().fit(X2, Y2, intercept_init=np.zeros((3,)))

    def test_sgd_proba(self):
        # Check SGD.predict_proba

        # Hinge loss does not allow for conditional prob estimate.
        # We cannot use the factory here, because it defines predict_proba
        # anyway.
        clf = SGDClassifier(loss="hinge", alpha=0.01, n_iter=10).fit(X, Y)
        assert_false(hasattr(clf, "predict_proba"))
        assert_false(hasattr(clf, "predict_log_proba"))

        # log and modified_huber losses can output probability estimates
        # binary case
        for loss in ["log", "modified_huber"]:
            clf = self.factory(loss="modified_huber", alpha=0.01, n_iter=10)
            clf.fit(X, Y)
            p = clf.predict_proba([3, 2])
            assert_true(p[0, 1] > 0.5)
            p = clf.predict_proba([-1, -1])
            assert_true(p[0, 1] < 0.5)

            p = clf.predict_log_proba([3, 2])
            assert_true(p[0, 1] > p[0, 0])
            p = clf.predict_log_proba([-1, -1])
            assert_true(p[0, 1] < p[0, 0])

        # log loss multiclass probability estimates
        clf = self.factory(loss="log", alpha=0.01, n_iter=10).fit(X2, Y2)

        d = clf.decision_function([[.1, -.1], [.3, .2]])
        p = clf.predict_proba([[.1, -.1], [.3, .2]])
        assert_array_equal(np.argmax(p, axis=1), np.argmax(d, axis=1))
        assert_almost_equal(p[0].sum(), 1)
        assert_true(np.all(p[0] >= 0))

        p = clf.predict_proba([-1, -1])
        d = clf.decision_function([-1, -1])
        assert_array_equal(np.argsort(p[0]), np.argsort(d[0]))

        l = clf.predict_log_proba([3, 2])
        p = clf.predict_proba([3, 2])
        assert_array_almost_equal(np.log(p), l)

        l = clf.predict_log_proba([-1, -1])
        p = clf.predict_proba([-1, -1])
        assert_array_almost_equal(np.log(p), l)

        # Modified Huber multiclass probability estimates; requires a separate
        # test because the hard zero/one probabilities may destroy the
        # ordering present in decision_function output.
        clf = self.factory(loss="modified_huber", alpha=0.01, n_iter=10)
        clf.fit(X2, Y2)
        d = clf.decision_function([3, 2])
        p = clf.predict_proba([3, 2])
        if not isinstance(self, SparseSGDClassifierTestCase):
            assert_equal(np.argmax(d, axis=1), np.argmax(p, axis=1))
        else:   # XXX the sparse test gets a different X2 (?)
            assert_equal(np.argmin(d, axis=1), np.argmin(p, axis=1))

        # the following sample produces decision_function values < -1,
        # which would cause naive normalization to fail (see comment
        # in SGDClassifier.predict_proba)
        x = X.mean(axis=0)
        d = clf.decision_function(x)
        if np.all(d < -1):  # XXX not true in sparse test case (why?)
            p = clf.predict_proba(x)
            assert_array_almost_equal(p[0], [1 / 3.] * 3)

    def test_sgd_l1(self):
        # Test L1 regularization
        n = len(X4)
        rng = np.random.RandomState(13)
        idx = np.arange(n)
        rng.shuffle(idx)

        X = X4[idx, :]
        Y = Y4[idx]

        clf = self.factory(penalty='l1', alpha=.2, fit_intercept=False,
                           n_iter=2000, shuffle=False)
        clf.fit(X, Y)
        assert_array_equal(clf.coef_[0, 1:-1], np.zeros((4,)))
        pred = clf.predict(X)
        assert_array_equal(pred, Y)

        # test sparsify with dense inputs
        clf.sparsify()
        assert_true(sp.issparse(clf.coef_))
        pred = clf.predict(X)
        assert_array_equal(pred, Y)

        # pickle and unpickle with sparse coef_
        clf = pickle.loads(pickle.dumps(clf))
        assert_true(sp.issparse(clf.coef_))
        pred = clf.predict(X)
        assert_array_equal(pred, Y)

    def test_class_weights(self):
        # Test class weights.
        X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                      [1.0, 1.0], [1.0, 0.0]])
        y = [1, 1, 1, -1, -1]

        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False,
                           class_weight=None)
        clf.fit(X, y)
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([1]))

        # we give a small weights to class 1
        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False,
                           class_weight={1: 0.001})
        clf.fit(X, y)

        # now the hyperplane should rotate clock-wise and
        # the prediction on this point should shift
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1]))

    def test_equal_class_weight(self):
        # Test if equal class weights approx. equals no class weights.
        X = [[1, 0], [1, 0], [0, 1], [0, 1]]
        y = [0, 0, 1, 1]
        clf = self.factory(alpha=0.1, n_iter=1000, class_weight=None)
        clf.fit(X, y)

        X = [[1, 0], [0, 1]]
        y = [0, 1]
        clf_weighted = self.factory(alpha=0.1, n_iter=1000,
                                    class_weight={0: 0.5, 1: 0.5})
        clf_weighted.fit(X, y)

        # should be similar up to some epsilon due to learning rate schedule
        assert_almost_equal(clf.coef_, clf_weighted.coef_, decimal=2)

    @raises(ValueError)
    def test_wrong_class_weight_label(self):
        # ValueError due to not existing class label.
        clf = self.factory(alpha=0.1, n_iter=1000, class_weight={0: 0.5})
        clf.fit(X, Y)

    @raises(ValueError)
    def test_wrong_class_weight_format(self):
        # ValueError due to wrong class_weight argument type.
        clf = self.factory(alpha=0.1, n_iter=1000, class_weight=[0.5])
        clf.fit(X, Y)

    def test_weights_multiplied(self):
        # Tests that class_weight and sample_weight are multiplicative
        class_weights = {1: .6, 2: .3}
        sample_weights = np.random.random(Y4.shape[0])
        multiplied_together = np.copy(sample_weights)
        multiplied_together[Y4 == 1] *= class_weights[1]
        multiplied_together[Y4 == 2] *= class_weights[2]

        clf1 = self.factory(alpha=0.1, n_iter=20, class_weight=class_weights)
        clf2 = self.factory(alpha=0.1, n_iter=20)

        clf1.fit(X4, Y4, sample_weight=sample_weights)
        clf2.fit(X4, Y4, sample_weight=multiplied_together)

        assert_almost_equal(clf1.coef_, clf2.coef_)

    def test_balanced_weight(self):
        # Test class weights for imbalanced data"""
        # compute reference metrics on iris dataset that is quite balanced by
        # default
        X, y = iris.data, iris.target
        X = scale(X)
        idx = np.arange(X.shape[0])
        rng = np.random.RandomState(6)
        rng.shuffle(idx)
        X = X[idx]
        y = y[idx]
        clf = self.factory(alpha=0.0001, n_iter=1000,
                           class_weight=None, shuffle=False).fit(X, y)
        assert_almost_equal(metrics.f1_score(y, clf.predict(X), average='weighted'), 0.96,
                            decimal=1)

        # make the same prediction using balanced class_weight
        clf_balanced = self.factory(alpha=0.0001, n_iter=1000,
                                    class_weight="balanced",
                                    shuffle=False).fit(X, y)
        assert_almost_equal(metrics.f1_score(y, clf_balanced.predict(X), average='weighted'), 0.96,
                            decimal=1)

        # Make sure that in the balanced case it does not change anything
        # to use "balanced"
        assert_array_almost_equal(clf.coef_, clf_balanced.coef_, 6)

        # build an very very imbalanced dataset out of iris data
        X_0 = X[y == 0, :]
        y_0 = y[y == 0]

        X_imbalanced = np.vstack([X] + [X_0] * 10)
        y_imbalanced = np.concatenate([y] + [y_0] * 10)

        # fit a model on the imbalanced data without class weight info
        clf = self.factory(n_iter=1000, class_weight=None, shuffle=False)
        clf.fit(X_imbalanced, y_imbalanced)
        y_pred = clf.predict(X)
        assert_less(metrics.f1_score(y, y_pred, average='weighted'), 0.96)

        # fit a model with balanced class_weight enabled
        clf = self.factory(n_iter=1000, class_weight="balanced", shuffle=False)
        clf.fit(X_imbalanced, y_imbalanced)
        y_pred = clf.predict(X)
        assert_greater(metrics.f1_score(y, y_pred, average='weighted'), 0.96)

        # fit another using a fit parameter override
        clf = self.factory(n_iter=1000, class_weight="balanced", shuffle=False)
        clf.fit(X_imbalanced, y_imbalanced)
        y_pred = clf.predict(X)
        assert_greater(metrics.f1_score(y, y_pred, average='weighted'), 0.96)

    def test_sample_weights(self):
        # Test weights on individual samples
        X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
                      [1.0, 1.0], [1.0, 0.0]])
        y = [1, 1, 1, -1, -1]

        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False)
        clf.fit(X, y)
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([1]))

        # we give a small weights to class 1
        clf.fit(X, y, sample_weight=[0.001] * 3 + [1] * 2)

        # now the hyperplane should rotate clock-wise and
        # the prediction on this point should shift
        assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1]))

    @raises(ValueError)
    def test_wrong_sample_weights(self):
        # Test if ValueError is raised if sample_weight has wrong shape
        clf = self.factory(alpha=0.1, n_iter=1000, fit_intercept=False)
        # provided sample_weight too long
        clf.fit(X, Y, sample_weight=np.arange(7))

    @raises(ValueError)
    def test_partial_fit_exception(self):
        clf = self.factory(alpha=0.01)
        # classes was not specified
        clf.partial_fit(X3, Y3)

    def test_partial_fit_binary(self):
        third = X.shape[0] // 3
        clf = self.factory(alpha=0.01)
        classes = np.unique(Y)

        clf.partial_fit(X[:third], Y[:third], classes=classes)
        assert_equal(clf.coef_.shape, (1, X.shape[1]))
        assert_equal(clf.intercept_.shape, (1,))
        assert_equal(clf.decision_function([0, 0]).shape, (1, ))
        id1 = id(clf.coef_.data)

        clf.partial_fit(X[third:], Y[third:])
        id2 = id(clf.coef_.data)
        # check that coef_ haven't been re-allocated
        assert_true(id1, id2)

        y_pred = clf.predict(T)
        assert_array_equal(y_pred, true_result)

    def test_partial_fit_multiclass(self):
        third = X2.shape[0] // 3
        clf = self.factory(alpha=0.01)
        classes = np.unique(Y2)

        clf.partial_fit(X2[:third], Y2[:third], classes=classes)
        assert_equal(clf.coef_.shape, (3, X2.shape[1]))
        assert_equal(clf.intercept_.shape, (3,))
        assert_equal(clf.decision_function([0, 0]).shape, (1, 3))
        id1 = id(clf.coef_.data)

        clf.partial_fit(X2[third:], Y2[third:])
        id2 = id(clf.coef_.data)
        # check that coef_ haven't been re-allocated
        assert_true(id1, id2)

    def test_fit_then_partial_fit(self):
        # Partial_fit should work after initial fit in the multiclass case.
        # Non-regression test for #2496; fit would previously produce a
        # Fortran-ordered coef_ that subsequent partial_fit couldn't handle.
        clf = self.factory()
        clf.fit(X2, Y2)
        clf.partial_fit(X2, Y2)     # no exception here

    def _test_partial_fit_equal_fit(self, lr):
        for X_, Y_, T_ in ((X, Y, T), (X2, Y2, T2)):
            clf = self.factory(alpha=0.01, eta0=0.01, n_iter=2,
                               learning_rate=lr, shuffle=False)
            clf.fit(X_, Y_)
            y_pred = clf.decision_function(T_)
            t = clf.t_

            classes = np.unique(Y_)
            clf = self.factory(alpha=0.01, eta0=0.01, learning_rate=lr,
                               shuffle=False)
            for i in range(2):
                clf.partial_fit(X_, Y_, classes=classes)
            y_pred2 = clf.decision_function(T_)

            assert_equal(clf.t_, t)
            assert_array_almost_equal(y_pred, y_pred2, decimal=2)

    def test_partial_fit_equal_fit_constant(self):
        self._test_partial_fit_equal_fit("constant")

    def test_partial_fit_equal_fit_optimal(self):
        self._test_partial_fit_equal_fit("optimal")

    def test_partial_fit_equal_fit_invscaling(self):
        self._test_partial_fit_equal_fit("invscaling")

    def test_regression_losses(self):
        clf = self.factory(alpha=0.01, learning_rate="constant",
                           eta0=0.1, loss="epsilon_insensitive")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

        clf = self.factory(alpha=0.01, learning_rate="constant",
                           eta0=0.1, loss="squared_epsilon_insensitive")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

        clf = self.factory(alpha=0.01, loss="huber")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

        clf = self.factory(alpha=0.01, learning_rate="constant", eta0=0.01,
                           loss="squared_loss")
        clf.fit(X, Y)
        assert_equal(1.0, np.mean(clf.predict(X) == Y))

    def test_warm_start_multiclass(self):
        self._test_warm_start(X2, Y2, "optimal")

    def test_multiple_fit(self):
        # Test multiple calls of fit w/ different shaped inputs.
        clf = self.factory(alpha=0.01, n_iter=5,
                           shuffle=False)
        clf.fit(X, Y)
        assert_true(hasattr(clf, "coef_"))

        # Non-regression test: try fitting with a different label set.
        y = [["ham", "spam"][i] for i in LabelEncoder().fit_transform(Y)]
        clf.fit(X[:, :-1], y)

class SparseSGDClassifierTestCase(DenseSGDClassifierTestCase):
    """Run exactly the same tests using the sparse representation variant"""

    factory_class = SparseSGDClassifier

###############################################################################
# Regression Test Case

class DenseSGDRegressorTestCase(unittest.TestCase, CommonTest):
    """Test suite for the dense representation variant of SGD"""

    factory_class = SGDRegressor

    def test_sgd(self):
        # Check that SGD gives any results.
        clf = self.factory(alpha=0.1, n_iter=2,
                           fit_intercept=False)
        clf.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
        assert_equal(clf.coef_[0], clf.coef_[1])

    @raises(ValueError)
    def test_sgd_bad_penalty(self):
        # Check whether expected ValueError on bad penalty
        self.factory(penalty='foobar', l1_ratio=0.85)

    @raises(ValueError)
    def test_sgd_bad_loss(self):
        # Check whether expected ValueError on bad loss
        self.factory(loss="foobar")

    def test_sgd_averaged_computed_correctly(self):
        # Tests the average regressor matches the naive implementation

        eta = .001
        alpha = .01
        n_samples = 20
        n_features = 10
        rng = np.random.RandomState(0)
        X = rng.normal(size=(n_samples, n_features))
        w = rng.normal(size=n_features)

        # simple linear function without noise
        y = np.dot(X, w)

        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        clf.fit(X, y)
        average_weights, average_intercept = self.asgd(X, y, eta, alpha)

        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=16)
        assert_almost_equal(clf.intercept_, average_intercept, decimal=16)

    def test_sgd_averaged_partial_fit(self):
        # Tests whether the partial fit yields the same average as the fit
        eta = .001
        alpha = .01
        n_samples = 20
        n_features = 10
        rng = np.random.RandomState(0)
        X = rng.normal(size=(n_samples, n_features))
        w = rng.normal(size=n_features)

        # simple linear function without noise
        y = np.dot(X, w)

        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        clf.partial_fit(X[:int(n_samples / 2)][:], y[:int(n_samples / 2)])
        clf.partial_fit(X[int(n_samples / 2):][:], y[int(n_samples / 2):])
        average_weights, average_intercept = self.asgd(X, y, eta, alpha)

        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=16)
        assert_almost_equal(clf.intercept_[0], average_intercept, decimal=16)

    def test_average_sparse(self):
        # Checks the average weights on data with 0s

        eta = .001
        alpha = .01
        clf = self.factory(loss='squared_loss',
                           learning_rate='constant',
                           eta0=eta, alpha=alpha,
                           fit_intercept=True,
                           n_iter=1, average=True, shuffle=False)

        n_samples = Y3.shape[0]

        clf.partial_fit(X3[:int(n_samples / 2)][:], Y3[:int(n_samples / 2)])
        clf.partial_fit(X3[int(n_samples / 2):][:], Y3[int(n_samples / 2):])
        average_weights, average_intercept = self.asgd(X3, Y3, eta, alpha)

        assert_array_almost_equal(clf.coef_,
                                  average_weights,
                                  decimal=16)
        assert_almost_equal(clf.intercept_, average_intercept, decimal=16)

    def test_sgd_least_squares_fit(self):
        xmin, xmax = -5, 5
        n_samples = 100
        rng = np.random.RandomState(0)
        X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1)

        # simple linear function without noise
        y = 0.5 * X.ravel()

        clf = self.factory(loss='squared_loss', alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.99)

        # simple linear function with noise
        y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()

        clf = self.factory(loss='squared_loss', alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.5)

    def test_sgd_epsilon_insensitive(self):
        xmin, xmax = -5, 5
        n_samples = 100
        X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1)

        # simple linear function without noise
        y = 0.5 * X.ravel()

        clf = self.factory(loss='epsilon_insensitive', epsilon=0.01,
                           alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_true(score > 0.99)

        # simple linear function with noise
        y = 0.5 * X.ravel() \
            + np.random.randn(n_samples, 1).ravel()

        clf = self.factory(loss='epsilon_insensitive', epsilon=0.01,
                           alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_true(score > 0.5)

    def test_sgd_huber_fit(self):
        xmin, xmax = -5, 5
        n_samples = 100
        rng = np.random.RandomState(0)
        X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1)

        # simple linear function without noise
        y = 0.5 * X.ravel()

        clf = self.factory(loss="huber", epsilon=0.1, alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.99)

        # simple linear function with noise
        y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()

        clf = self.factory(loss="huber", epsilon=0.1, alpha=0.1, n_iter=20,
                           fit_intercept=False)
        clf.fit(X, y)
        score = clf.score(X, y)
        assert_greater(score, 0.5)

    def test_elasticnet_convergence(self):
        # Check that the SGD output is consistent with coordinate descent

        n_samples, n_features = 1000, 5
        rng = np.random.RandomState(0)
        X = np.random.randn(n_samples, n_features)
        # ground_truth linear model that generate y from X and to which the
        # models should converge if the regularizer would be set to 0.0
        ground_truth_coef = rng.randn(n_features)
        y = np.dot(X, ground_truth_coef)

        # XXX: alpha = 0.1 seems to cause convergence problems
        for alpha in [0.01, 0.001]:
            for l1_ratio in [0.5, 0.8, 1.0]:
                cd = linear_model.ElasticNet(alpha=alpha, l1_ratio=l1_ratio,
                                             fit_intercept=False)
                cd.fit(X, y)
                sgd = self.factory(penalty='elasticnet', n_iter=50,
                                   alpha=alpha, l1_ratio=l1_ratio,
                                   fit_intercept=False)
                sgd.fit(X, y)
                err_msg = ("cd and sgd did not converge to comparable "
                           "results for alpha=%f and l1_ratio=%f"
                           % (alpha, l1_ratio))
                assert_almost_equal(cd.coef_, sgd.coef_, decimal=2,
                                    err_msg=err_msg)

    def test_partial_fit(self):
        third = X.shape[0] // 3
        clf = self.factory(alpha=0.01)

        clf.partial_fit(X[:third], Y[:third])
        assert_equal(clf.coef_.shape, (X.shape[1], ))
        assert_equal(clf.intercept_.shape, (1,))
        assert_equal(clf.decision_function([0, 0]).shape, (1, ))
        id1 = id(clf.coef_.data)

        clf.partial_fit(X[third:], Y[third:])
        id2 = id(clf.coef_.data)
        # check that coef_ haven't been re-allocated
        assert_true(id1, id2)

    def _test_partial_fit_equal_fit(self, lr):
        clf = self.factory(alpha=0.01, n_iter=2, eta0=0.01,
                           learning_rate=lr, shuffle=False)
        clf.fit(X, Y)
        y_pred = clf.predict(T)
        t = clf.t_

        clf = self.factory(alpha=0.01, eta0=0.01,
                           learning_rate=lr, shuffle=False)
        for i in range(2):
            clf.partial_fit(X, Y)
        y_pred2 = clf.predict(T)

        assert_equal(clf.t_, t)
        assert_array_almost_equal(y_pred, y_pred2, decimal=2)

    def test_partial_fit_equal_fit_constant(self):
        self._test_partial_fit_equal_fit("constant")

    def test_partial_fit_equal_fit_optimal(self):
        self._test_partial_fit_equal_fit("optimal")

    def test_partial_fit_equal_fit_invscaling(self):
        self._test_partial_fit_equal_fit("invscaling")

    def test_loss_function_epsilon(self):
        clf = self.factory(epsilon=0.9)
        clf.set_params(epsilon=0.1)
        assert clf.loss_functions['huber'][1] == 0.1

class SparseSGDRegressorTestCase(DenseSGDRegressorTestCase):
    # Run exactly the same tests using the sparse representation variant

    factory_class = SparseSGDRegressor

def test_l1_ratio():
    # Test if l1 ratio extremes match L1 and L2 penalty settings.
    X, y = datasets.make_classification(n_samples=1000,
                                        n_features=100, n_informative=20,
                                        random_state=1234)

    # test if elasticnet with l1_ratio near 1 gives same result as pure l1
    est_en = SGDClassifier(alpha=0.001, penalty='elasticnet',
                           l1_ratio=0.9999999999, random_state=42).fit(X, y)
    est_l1 = SGDClassifier(alpha=0.001, penalty='l1', random_state=42).fit(X, y)
    assert_array_almost_equal(est_en.coef_, est_l1.coef_)

    # test if elasticnet with l1_ratio near 0 gives same result as pure l2
    est_en = SGDClassifier(alpha=0.001, penalty='elasticnet',
                           l1_ratio=0.0000000001, random_state=42).fit(X, y)
    est_l2 = SGDClassifier(alpha=0.001, penalty='l2', random_state=42).fit(X, y)
    assert_array_almost_equal(est_en.coef_, est_l2.coef_)

def test_underflow_or_overlow():
    with np.errstate(all='raise'):
        # Generate some weird data with hugely unscaled features
        rng = np.random.RandomState(0)
        n_samples = 100
        n_features = 10

        X = rng.normal(size=(n_samples, n_features))
        X[:, :2] *= 1e300
        assert_true(np.isfinite(X).all())

        # Use MinMaxScaler to scale the data without introducing a numerical
        # instability (computing the standard deviation naively is not possible
        # on this data)
        X_scaled = MinMaxScaler().fit_transform(X)
        assert_true(np.isfinite(X_scaled).all())

        # Define a ground truth on the scaled data
        ground_truth = rng.normal(size=n_features)
        y = (np.dot(X_scaled, ground_truth) > 0.).astype(np.int32)
        assert_array_equal(np.unique(y), [0, 1])

        model = SGDClassifier(alpha=0.1, loss='squared_hinge', n_iter=500)

        # smoke test: model is stable on scaled data
        model.fit(X_scaled, y)
        assert_true(np.isfinite(model.coef_).all())

        # model is numerically unstable on unscaled data
        msg_regxp = (r"Floating-point under-/overflow occurred at epoch #.*"
                     " Scaling input data with StandardScaler or MinMaxScaler"
                     " might help.")
        assert_raises_regexp(ValueError, msg_regxp, model.fit, X, y)

def test_numerical_stability_large_gradient():
    # Non regression test case for numerical stability on scaled problems
    # where the gradient can still explode with some losses
    model = SGDClassifier(loss='squared_hinge', n_iter=10, shuffle=True,
                          penalty='elasticnet', l1_ratio=0.3, alpha=0.01,
                          eta0=0.001, random_state=0)
    with np.errstate(all='raise'):
        model.fit(iris.data, iris.target)
    assert_true(np.isfinite(model.coef_).all())

def test_large_regularization():
    # Non regression tests for numerical stability issues caused by large
    # regularization parameters
    for penalty in ['l2', 'l1', 'elasticnet']:
        model = SGDClassifier(alpha=1e5, learning_rate='constant', eta0=0.1,
                              n_iter=5, penalty=penalty, shuffle=False)
        with np.errstate(all='raise'):
            model.fit(iris.data, iris.target)
        assert_array_almost_equal(model.coef_, np.zeros_like(model.coef_))


# -*- coding: utf-8 -*-

import pandas as pd
import scrapy
from scrapy import Request
from scrapy import Selector
from scrapy import signals

from fooltrader.contract.files_contract import get_kdata_path
from fooltrader.utils.utils import index_df_with_time, to_time_str, to_float

class Sp500Spider(scrapy.Spider):
    name = "sp500_spider"

    def __init__(self, name=None, **kwargs):
        super().__init__(name, **kwargs)
        self.security_item = {'id': 'index_nasdaq_sp500',
                              'code': 'SP500',
                              'name': 'SP500',
                              'listDate': '1871-01-01',
                              'timestamp': '1871-01-01',
                              'exchange': 'nasdaq',
                              'type': 'index'}
        self.df_close = pd.DataFrame()
        self.df_pe = pd.DataFrame()

    def start_requests(self):
        pe_url = 'http://www.multpl.com/table?f=m'
        price_url = 'http://www.multpl.com/s-p-500-historical-prices/table/by-month'

        yield Request(url=pe_url,
                      callback=self.download_sp500_pe)

        yield Request(url=price_url,
                      callback=self.download_sp500_price)

    def download_sp500_price(self, response):
        trs = response.xpath('//*[@id="datatable"]/tr').extract()

        price_jsons = []

        try:
            for tr in trs[1:]:
                tds = Selector(text=tr).xpath('//td//text()').extract()
                tds = [x.strip() for x in tds if x.strip()]

                price_jsons.append({"timestamp": to_time_str(tds[0]),
                                    "close": to_float(tds[1])})

            if price_jsons:
                self.df_close = self.df_close.append(price_jsons, ignore_index=True)
                self.df_close = index_df_with_time(self.df_close)
        except Exception as e:
            self.logger.exception('error when getting sp500 price url={} error={}'.format(response.url, e))

    def download_sp500_pe(self, response):
        trs = response.xpath('//*[@id="datatable"]/tr').extract()

        price_jsons = []

        try:
            for tr in trs[1:]:
                tds = Selector(text=tr).xpath('//td//text()').extract()
                tds = [x.strip() for x in tds if x.strip()]

                price_jsons.append({"timestamp": to_time_str(tds[0]),
                                    "pe": to_float(tds[1])})

            if price_jsons:
                self.df_pe = self.df_pe.append(price_jsons, ignore_index=True)
                self.df_pe = index_df_with_time(self.df_pe)
        except Exception as e:
            self.logger.exception('error when getting sp500 pe url={} error={}'.format(response.url, e))

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(Sp500Spider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def spider_closed(self, spider, reason):
        self.df_pe['close'] = self.df_close['close']
        self.df_pe['code'] = self.security_item['code']
        self.df_pe['securityId'] = self.security_item['id']
        self.df_pe['name'] = self.security_item['name']
        self.df_pe.to_csv(get_kdata_path(self.security_item), index=False)
        spider.logger.info('Spider closed: %s,%s\n', spider.name, reason)


"""
Test Scikit-learn
"""
import numpy as np
from mlens.ensemble import SuperLearner, Subsemble, BlendEnsemble, TemporalEnsemble
from mlens.testing.dummy import return_pickled
try:
    from sklearn.utils.estimator_checks import check_estimator
    from sklearn.linear_model import Lasso, LinearRegression
    from sklearn.neighbors import KNeighborsRegressor
    from sklearn.svm import SVR
    from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor

    from sklearn.decomposition import PCA
    from sklearn.preprocessing import StandardScaler, PolynomialFeatures

    from sklearn.datasets import load_boston

    has_sklearn = True

except ImportError:
    has_sklearn = False

if has_sklearn:

    X, y = load_boston(True)

    estimators = [Lasso(),
                  GradientBoostingRegressor(),
                  LinearRegression(),
                  KNeighborsRegressor(),
                  SVR(gamma='scale'),
                  RandomForestRegressor(n_estimators=100),
                  ]

    est_prep = {'prep1': estimators,
                'prep2': estimators,
                'prep3': estimators}

    prep_1 = [PCA()]
    prep_2 = [PolynomialFeatures(), StandardScaler()]

    prep = {'prep1': prep_1,
            'prep2': prep_2,
            'prep3': []}

    def get_ensemble(cls, backend, preprocessing, **kwargs):
        """Get ensemble."""
        if preprocessing:
            est = est_prep
        else:
            est = estimators
        ens = cls(backend=backend, **kwargs)
        ens.add(est, preprocessing)
        ens.add(LinearRegression(), meta=True)
        return ens

    def test_super_learner_s_m():
        """[SuperLearner] Test scikit-learn comp - mp | np"""
        ens = get_ensemble(SuperLearner, 'multiprocessing', None)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_super_learner_f_m():
        """[SuperLearner] Test scikit-learn comp - mp | p"""
        ens = get_ensemble(SuperLearner, 'multiprocessing', prep)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_super_learner_s_t():
        """[SuperLearner] Test scikit-learn comp - th | np"""
        ens = get_ensemble(SuperLearner, 'threading', None)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_super_learner_f_t():
        """[SuperLearner] Test scikit-learn comp - th | p"""
        ens = get_ensemble(SuperLearner, 'threading', prep)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_subsemble_s_m():
        """[Subsemble] Test scikit-learn comp - mp | np"""
        ens = get_ensemble(Subsemble, 'multiprocessing', None)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_subsemble_f_m():
        """[Subsemble] Test scikit-learn comp - mp | p"""
        ens = get_ensemble(Subsemble, 'multiprocessing', prep)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_subsemble_s_t():
        """[Subsemble] Test scikit-learn comp - th | np"""
        ens = get_ensemble(Subsemble, 'threading', None)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_subsemble_f_t():
        """[Subsemble] Test scikit-learn comp - th | p"""
        ens = get_ensemble(Subsemble, 'threading', prep)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_blend_s_m():
        """[BlendEnsemble] Test scikit-learn comp - mp | np"""
        ens = get_ensemble(BlendEnsemble, 'multiprocessing', None)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_blend_f_m():
        """[BlendEnsemble] Test scikit-learn comp - mp | p"""
        ens = get_ensemble(BlendEnsemble, 'multiprocessing', prep)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_blend_s_m():
        """[BlendEnsemble] Test scikit-learn comp - th | np"""
        ens = get_ensemble(BlendEnsemble, 'threading', None)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_blend_f_m():
        """[BlendEnsemble] Test scikit-learn comp - th | p"""
        ens = get_ensemble(BlendEnsemble, 'threading', prep)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_temporal_s_m():
        """[TemporalEnsemble] Test scikit-learn comp - mp | np"""
        ens = get_ensemble(TemporalEnsemble, 'multiprocessing', None, step_size=10)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_temporal_f_m():
        """[TemporalEnsemble] Test scikit-learn comp - mp | p"""
        ens = get_ensemble(TemporalEnsemble, 'multiprocessing', prep, step_size=10)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_temporal_s_m():
        """[TemporalEnsemble] Test scikit-learn comp - th | np"""
        ens = get_ensemble(TemporalEnsemble, 'threading', None, step_size=10)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)

    def test_temporal_f_m():
        """[TemporalEnsemble] Test scikit-learn comp - th | p"""
        ens = get_ensemble(TemporalEnsemble, 'threading', prep, step_size=10)
        ens.fit(X, y)
        p = ens.predict(X)
        assert p.shape == y.shape
        assert p.dtype == ens.layer_1.dtype

        ens = return_pickled(ens)
        pp = ens.predict(X)
        np.testing.assert_array_equal(p, pp)


# -*- coding: utf-8 -*-

"""
Evaluation Models
=================

"""

from __future__ import division

from copy import copy
from itertools import izip
from collections import defaultdict

import numpy as np
import pandas as pd

import tools

__all__ = (
    'DummyPriorModel',
    'EloModel',
    'EloResponseTime',
    'PFAModel',
    'PFAResponseTime',
    'PFAExt',
    'PFAExtTiming',
    'PFAExtStaircase',
    'PFAExtSpacing',
    'PFAGong',
    'PFAGongTiming',
    'PFATiming',
)

#: Dictionary of the most commonly used time effect functions in this thesis.
time_effect_funcs = {}

def register_time_effect(name):
    """Registers new time effect functions."""
    def register(time_effect):
        time_effect_funcs[name] = time_effect
    return register

@register_time_effect('log')
def time_effect_log(t, a=1.8, c=0.123):
    return a - c * np.log(t)

@register_time_effect('pow')
def time_effect_div(t, a=2, c=0.2):
    return a / (t+1) ** c

@register_time_effect('exp')
def time_effect_exp(t, a=1.6, c=0.01):
    return a * np.exp(-c * np.sqrt(t))

def init_time_effect(obj, name, parameters=('a', 'c')):
    """Prepares time effect function based on name. Initializes
    the given object with default parameters `a` and `c`.

    :param obj: Object to initialize with time effect function.
    :param name: Name of the time effect function.
    """
    time_effect_fun = time_effect_funcs[name]
    defaults = time_effect_fun.func_defaults

    a, c = parameters

    if getattr(obj, a, None) is None:
        setattr(obj, a, defaults[0])
    if getattr(obj, c, None) is None:
        setattr(obj, c, defaults[1])

    def time_effect(t):
        a_val, c_val = getattr(obj, a), getattr(obj, c)
        return time_effect_fun(t, a_val, c_val)

    return time_effect

class Question(object):
    """Representation of a question."""

    def __init__(self, **kwargs):
        self.id = kwargs.pop('id')
        self.user_id = kwargs.pop('user_id')
        self.place_id = kwargs.pop('place_id')
        self.type = kwargs.pop('type')
        self.inserted = kwargs.pop('inserted')
        self.options = kwargs.pop('options')

class Answer(Question):
    """Answer to a question."""

    def __init__(self, **kwargs):
        super(Answer, self).__init__(**kwargs)

        self.place_answered = kwargs.pop('place_answered')
        self.response_time = kwargs.pop('response_time')
        self.is_correct = kwargs.pop('is_correct')

class User(object):
    """Returns a user with given ID.

    :param user_id: ID of the user.
    :type user_id: int
    """
    def __init__(self, user_id):
        self.id = user_id
        self.skill_increments = []

    @property
    def skill(self):
        """Skill of the user."""
        return sum(self.skill_increments)

    @property
    def answers_count(self):
        """Number of answer of the user (equal to the number of
        skill increments.
        """
        return len(self.skill_increments)

    def inc_skill(self, increment):
        """Increments the skill of the user.

        :param increment: Increment (or decrement) of the skill.
        :type increment: int
        """
        self.skill_increments += [increment]

class Place(object):
    """Returns a place with given ID.

    :param place_id: ID of the place.
    :type place_id: int
    """
    def __init__(self, place_id):
        self.id = place_id
        self.difficulty_increments = []

    @property
    def difficulty(self):
        """Difficulty of the place."""
        return sum(self.difficulty_increments)

    @property
    def answers_count(self):
        """Number of answer for the place (equal to the number of
        difficulty increments.
        """
        return len(self.difficulty_increments)

    def inc_difficulty(self, increment):
        """Increments the difficulty of the place.

        :param increment: Increment (or decrement) of the difficulty.
        :type increment: int
        """
        self.difficulty_increments += [increment]

class Item(object):
    """Item representation.

    :param prior: Prior skills of users and difficulties of places.
    :type prior: dictionary
    :param user_id: ID of the user.
    :type user_id: int
    :param place_id: ID of the place.
    :type place_id: int
    """

    def __init__(self, prior, user_id, place_id):
        self.prior = prior
        self.user_id = user_id
        self.place_id = place_id

        self.practices = []
        self.knowledge_increments = []

    @property
    def user(self):
        """User answering the item."""
        return self.prior.users[self.user_id]

    @property
    def place(self):
        """Place of the item being asked."""
        return self.prior.places[self.place_id]

    @property
    def knowledge(self):
        """Knowledge of the item by the user."""
        return (
            (self.user.skill - self.place.difficulty)
            + sum(self.knowledge_increments)
        )

    @property
    def correct(self):
        """List of correct answers."""
        return [ans for ans in self.practices if ans.is_correct]

    @property
    def incorrect(self):
        """List of incorrect answers."""
        return [ans for ans in self.practices if not ans.is_correct]

    @property
    def last_inserted(self):
        """Returns the time of the last answer for this item
        or :obj:`None` if the item was never answered before.
        """
        if self.practices:
            return self.practices[-1].inserted

    @property
    def any_incorrect(self):
        """:obj:`True` if at least one of the practiced item
        was answered incorrectly, otherwise :obj:`False`.
        """
        return any(not answer.is_correct for answer in self.practices)

    def get_diffs(self, current):
        """Returns list of previous practices expresed as the number
        of seconds that passed between *current* practice and all
        the *previous* practices.

        :param current: Datetime of the current practice.
        :type place: string
        """
        return [
            tools.time_diff(current, prior.inserted)
            for prior in self.practices
        ]

    def inc_knowledge(self, increment):
        """Increments the knowledge of the user of the item.

        :param increment: Increment (or decrement) of the knowledge.
        :type increment: int
        """
        self.knowledge_increments += [increment]

    def add_practice(self, answer):
        """Registers new practice of the item.

        :param answer: Information about the answer.
        :type answer: :class:`pandas.Series` or :class:`Answer`
        """
        if isinstance(answer, pd.Series):
            self.practices += [Answer(**answer.to_dict())]
        else:
            self.practices += [copy(answer)]

class Model(object):
    """Abstract model class."""

    ABBR = None

    def respect_guess(self, prediction, options):
        """Updates prediction with respect to guessing paramter.

        :param prediction: Prediction calculated so far.
        :type prediction: float
        :param options: Number of options in the multiple-choice question.
        :type options: int
        """
        if options:
            val = 1 / len(options)
            return val + (1 - val) * prediction
        else:
            return prediction

    def predict(self, question):
        """Returns probability of correct answer for given question.

        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        raise NotImplementedError()

    def update(self, answer):
        """Performes an update of skills, difficulties or knowledge.

        :param answer: Asked question.
        :type answer: :class:`pandas.Series` or :class:`Answer`
        """
        raise NotImplementedError()

    def train(self, data):
        """Trains the model on given data set.

        :param data: Data set on which to train the model.
        :type data: :class:`pandas.DataFrame`
        """
        raise NotImplementedError()

    @classmethod
    def split_data(cls, data, ratio=0.7):
        """Classmethod that splits data into training set and test set.

        :param data: The object containing data.
        :type data: :class:`pandas.DataFrame`.
        :param ratio: What portion of data to include in the training set
            and the test set. :obj:`0.5` means that the data will be
            distributed equaly.
        :type ratio: float
        """
        raise NotImplementedError()

class DummyPriorModel(Model):
    """Dummy model that sets all skills of users and difficulties
    of places to zero.
    """

    class _User(object):
        """Returns a user with given ID."""

        def __init__(self, skill):
            self.skill = skill

    class _Place(object):
        """Returns a place with given ID."""

        def __init__(self, difficulty):
            self.difficulty = difficulty

    def __init__(self, skill=0.0, difficulty=0.0):
        self.users = defaultdict(lambda: self._User(skill))
        self.places = defaultdict(lambda: self._Place(difficulty))

    def update(self, answer):
        pass

    def train(self, data):
        pass

class EloModel(Model):
    """Predicts correctness of answers using Elo Rating System.
    The model is parametrized with `alpha` and `beta`. These parameters
    affect the uncertainty function.
    """
    ABBR = 'Elo'

    def __init__(self, alpha=1, beta=0.05):
        self.alpha = alpha
        self.beta = beta

        self.init_model()

    def init_model(self):
        """Initializes two attributes of the model. Both attributes are
        dataframes. The first attribute represents difficulties of countries.
        The second attribute represents global knowledge of students.
        """
        self.places = tools.keydefaultdict(Place)
        self.users = tools.keydefaultdict(User)

        self.predictions = {}

    def uncertainty(self, n):
        """Uncertainty function. The purpose is to make each update on
        the model trained with sequence of `n` answers less and less
        significant as the number of prior answers is bigger.

        :param n: Number of user's answers or total answers to a place.
        :type n: int
        """
        return self.alpha / (1 + self.beta * n)

    def predict(self, question):
        """Returns probability of correct answer for given question.

        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        user = self.users[question.user_id]
        place = self.places[question.place_id]

        prediction = tools.sigmoid(user.skill - place.difficulty)
        return self.respect_guess(prediction, question.options)

    def update(self, answer):
        """Updates skills of users and difficulties of places according
        to given answer.

        :param answer: Answer to a question.
        :type answer: :class:`pandas.Series`
        """
        user = self.users[answer.user_id]
        place = self.places[answer.place_id]

        prediction = self.predict(answer)
        shift = answer.is_correct - prediction

        user.inc_skill(self.uncertainty(user.answers_count) * shift)
        place.inc_difficulty(-(self.uncertainty(place.answers_count) * shift))

        self.predictions[answer.id] = prediction

    def train(self, data):
        """Trains the model on given data set.

        :param data: Data set on which to train the model.
        :type data: :class:`pandas.DataFrame`
        """
        self.init_model()
        data = tools.first_answers(data)
        data.sort(['inserted']).apply(self.update, axis=1)

    @classmethod
    def split_data(cls, data, ratio=0.7):
        """Classmethod that splits data into training set and test set.

        :param data: The object containing data.
        :type data: :class:`pandas.DataFrame`.
        :param ratio: What portion of data to include in the training set
            and the test set. :obj:`0.5` means that the data will be
            distributed equaly.
        :type ratio: float
        """
        data = tools.first_answers(data)
        return tools.split_data(data, ratio=ratio)

class EloResponseTime(EloModel):
    """Extension of the Elo model that takes response time of user
    into account.
    """
    ABBR = 'Elo/RT'

    def __init__(self, *args, **kwargs):
        self.zeta = kwargs.pop('zeta', 3)

        super(EloResponseTime, self).__init__(*args, **kwargs)

    def update(self, answer):
        """Updates skills of users and difficulties of places according
        to given answer.

        :param answer: Answer to a question.
        :type answer: :class:`pandas.Series` or :class:`Answer`
        """
        user = self.users[answer.user_id]
        place = self.places[answer.place_id]

        prediction = self.predict(answer)
        level = tools.automaticity_level(answer.response_time)

        prob = (prediction * self.zeta + level) / (self.zeta + 1)
        shift = answer.is_correct - prob

        user.inc_skill(self.uncertainty(user.answers_count) * shift)
        place.inc_difficulty(-(self.uncertainty(place.answers_count) * shift))

        self.predictions[answer.id] = prediction

class PFAModel(Model):
    """Standard Performance Factor Analysis.

    :param gamma: The significance of the update when the student
        answered correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answered incorrectly.
    :type delta: float
    """
    ABBR = 'PFA'

    def __init__(self, prior=None, gamma=3.4, delta=-0.3):
        super(PFAModel, self).__init__()

        self.prior = prior or DummyPriorModel()
        self.gamma = gamma
        self.delta = delta

        self.init_model()

    def init_model(self):
        """Initializes attribute of the model that stores current
        knowledge of places for all students.
        """
        self.items = tools.keydefaultdict(
            lambda *args: Item(self.prior, *args)
        )
        self.predictions = {}

    def predict(self, question):
        """Returns probability of correct answer for given question.

        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        item = self.items[question.user_id, question.place_id]

        knowledge = (
            item.knowledge +
            self.gamma * len(item.correct) +
            self.delta * len(item.incorrect)
        )

        return tools.sigmoid(knowledge)

    def update(self, answer):
        """Performes update of current knowledge of a user based on the
        given answer.

        :param answer: Answer to a question.
        :type answer: :class:`pandas.Series` or :class:`Answer`
        """
        item = self.items[answer.user_id, answer.place_id]

        if not item.practices:
            self.prior.update(answer)

        prediction = self.predict(answer)
        self.predictions[answer.id] = prediction

        item.add_practice(answer)

    def train(self, data):
        """Trains the model on given data set.

        :param data: Data set on which to train the model.
        :type data: :class:`pandas.DataFrame`
        """
        self.init_model()
        data.sort(['inserted']).apply(self.update, axis=1)

    @classmethod
    def split_data(self, data):
        """Classmethod that splits data into training set and test set.

        :param data: The object containing data.
        :type data: :class:`pandas.DataFrame`.
        """
        test_set = tools.last_answers(data)
        train_set = data[~data['id'].isin(test_set['id'])]

        return train_set, test_set

class PFAExt(PFAModel):
    """PFA model for estimation of current knowledge.

    :param gamma: The significance of the update when the student
        answered correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answered incorrectly.
    :type delta: float
    """
    ABBR = 'PFA/E'

    def predict(self, question):
        """Returns probability of correct answer for given question.

        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        item = self.items[question.user_id, question.place_id]
        prediction = tools.sigmoid(item.knowledge)
        return self.respect_guess(prediction, question.options)

    def update(self, answer):
        """Performes update of current knowledge of a user based on the
        given answer.

        :param answer: Answer to a question.
        :type answer: :class:`pandas.Series` or :class:`Answer`
        """
        item = self.items[answer.user_id, answer.place_id]

        if not item.practices:
            self.prior.update(answer)

        prediction = self.predict(answer)
        self.predictions[answer.id] = prediction

        item.add_practice(answer)

        if answer.is_correct:
            item.inc_knowledge(self.gamma * (1 - prediction))
        else:
            item.inc_knowledge(self.delta * prediction)

class PFAResponseTime(PFAExt):
    """An extended version of the PFAExt model which alters student's
    knowledge by respecting past response times.

    :param gamma: The significance of the update when the student
        answered correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answered incorrectly.
    :type delta: float
    :param zeta: The significance of response times.
    :type zeta: float
    """
    ABBR = 'PFA/E/RT'

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('gamma', 1.5)
        kwargs.setdefault('delta', -1.4)

        self.zeta = kwargs.pop('zeta', 1.9)

        super(PFAResponseTime, self).__init__(*args, **kwargs)

    def update(self, answer):
        """Performes update of current knowledge of a user based on the
        given answer.

        :param answer: Answer to a question.
        :type answer: :class:`pandas.Series` or :class:`Answer`
        """
        item = self.items[answer.user_id, answer.place_id]

        if not item.practices:
            self.prior.update(answer)

        prediction = self.predict(answer)
        self.predictions[answer.id] = prediction

        item.add_practice(answer)
        level = tools.automaticity_level(answer.response_time) / self.zeta

        if answer.is_correct:
            item.inc_knowledge(self.gamma * (1 - prediction) + level)
        else:
            item.inc_knowledge(self.delta * prediction + level)

class PFAExtTiming(PFAExt):
    """Alternative version of :class:`PFAExtSpacing` which ignores
    spacing effect. Only forgetting is considered.

    :param gamma: The significance of the update when the student
        answered correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answered incorrectly.
    :type delta: float
    :param time_effect_fun: Time effect function.
    :type time_effect_fun: callable or string
    """
    ABBR = 'PFA/E/T'

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('gamma', 2.3)
        kwargs.setdefault('delta', -0.9)

        time_effect = kwargs.pop('time_effect_fun', 'poly')

        if isinstance(time_effect, basestring):
            self.a, self.c = kwargs.pop('a', None), kwargs.pop('c', None)
            self.time_effect = init_time_effect(self, time_effect)
        else:
            self.time_effect = time_effect

        super(PFAExtTiming, self).__init__(*args, **kwargs)

    def predict(self, question):
        """Returns probability of correct answer for given question.

        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        item = self.items[question.user_id, question.place_id]

        if item.practices:
            seconds = tools.time_diff(question.inserted, item.last_inserted)
            time_effect = self.time_effect(seconds)
        else:
            time_effect = 0

        prediction = tools.sigmoid(item.knowledge + time_effect)
        return self.respect_guess(prediction, question.options)

class PFAExtStaircase(PFAExtTiming):
    """Alternative version of :class:`PFAESpacing` which ignores
    spacing effect. Only forgetting is considered given by staircase
    fucntion.

    :param gamma: The significance of the update when the student
        answered correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answered incorrectly.
    :type delta: float
    :param time_effect_fun: Values for staircase function.
    :type time_effect_fun: dict (tuples as keys)
    """
    ABBR = 'PFA/E/T staircase'

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('gamma', 2.5)
        kwargs.setdefault('delta', -0.8)

        self.staircase = tools.intervaldict(kwargs.pop('staircase'))
        self.time_effect = lambda k: self.staircase[k]

        super(PFAExtTiming, self).__init__(*args, **kwargs)

class PFAExtSpacing(PFAExtTiming):
    """Extended version of PFA that takes into account the effect of
    forgetting and spacing.

    :param gamma: The significance of the update when the student
        answers correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answers incorrectly.
    :type delta: float
    :param spacing_rate: The significance of the spacing effect. Lower
        values make the effect less significant. If the spacing rate
        is set to zero, the model is unaware of the spacing effect.
    :type spacing_rate: float
    :param decay_rate: The significance of the forgetting effect. Higher
        values of decay rate make the students forget the item faster
        and vice versa.
    :type decay_rate: float
    """
    ABBR = 'PFA/E/S'

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('gamma', 2.8)
        kwargs.setdefault('delta', -0.7)

        self.spacing_rate = kwargs.pop('spacing_rate', 0)
        self.decay_rate = kwargs.pop('decay_rate', 0.18)
        self.iota = kwargs.pop('iota', 1.5)

        super(PFAExtSpacing, self).__init__(*args, **kwargs)

    def memory_strength(self, question):
        """Estimates memory strength of an item.

        :param question: Asked question.
        :type question: :class:`pandas.Series`
        """
        item = self.items[question.user_id, question.place_id]
        practices = item.get_diffs(question.inserted)

        if len(practices) > 0:
            return self.iota + tools.memory_strength(
                filter(lambda x: x > 0, practices),
                spacing_rate=self.spacing_rate,
                decay_rate=self.decay_rate,
            )

    def predict(self, question):
        """Returns probability of correct answer for given question.

        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        item = self.items[question.user_id, question.place_id]

        if item.any_incorrect:
            strength = self.memory_strength(question)
        else:
            strength = 0

        prediction = tools.sigmoid(item.knowledge + strength)
        return self.respect_guess(prediction, question.options)

class PFAGong(PFAModel):
    """Yue Gong's extended Performance Factor Analysis.

    :param gamma: The significance of the update when the student
        answers correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answers incorrectly.
    :type delta: float
    :param decay: Decay rate of answers.
    :type decay: float
    """
    ABBR = 'PFA/G'

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('gamma', 2.1)
        kwargs.setdefault('delta', -0.8)

        self.decay = kwargs.pop('decay', 0.8)

        super(PFAGong, self).__init__(*args, **kwargs)

    def get_weights(self, item, question):
        """Returns weights of previous answers to the given item.

        :param item: *Item* (i.e. practiced place by a user).
        :type item: :class:`Item`
        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        correct_weights = [
            ans.is_correct * self.decay ** k for k, ans
            in tools.reverse_enumerate(item.practices)
        ]
        incorrect_weights = [
            (1 - ans.is_correct) * self.decay ** k for k, ans
            in tools.reverse_enumerate(item.practices)
        ]
        return sum(correct_weights), sum(incorrect_weights)

    def predict(self, question):
        """Returns probability of correct answer for given question.

        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        item = self.items[question.user_id, question.place_id]
        correct_weight, incorrect_weight = self.get_weights(item, question)

        knowledge = (
            item.knowledge +
            self.gamma * correct_weight +
            self.delta * incorrect_weight
        )

        prediction = tools.sigmoid(knowledge)
        return self.respect_guess(prediction, question.options)

    def update(self, answer):
        """Performes update of current knowledge of a user based on the
        given answer.

        :param answer: Answer to a question.
        :type answer: :class:`pandas.Series` or :class:`Answer`
        """
        item = self.items[answer.user_id, answer.place_id]

        if not item.practices:
            self.prior.update(answer)

        prediction = self.predict(answer)
        self.predictions[answer.id] = prediction

        item.add_practice(answer)

class PFAGongTiming(PFAGong):
    """Performance Factor Analysis combining some aspects of both
    the Yue Gong's PFA and the ACT-R model.

    :param gamma: The significance of the update when the student
        answers correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answers incorrectly.
    :type delta: float
    :param time_effect_fun: Time effect function.
    :type time_effect_fun: callable or string
    """
    ABBR = 'PFA/G/T old'

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('gamma', 1.7)
        kwargs.setdefault('delta', 0.5)

        time_effect = kwargs.pop('time_effect_fun', 'pow')

        if isinstance(time_effect, basestring):
            self.a, self.c = kwargs.pop('a', None), kwargs.pop('c', None)
            self.time_effect = init_time_effect(self, time_effect)
        else:
            self.time_effect = time_effect

        super(PFAGong, self).__init__(*args, **kwargs)

    def get_weights(self, item, question):
        """Returns weights of previous answers to the given item.

        :param item: *Item* (i.e. practiced place by a user).
        :type item: :class:`Item`
        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        correct_weights = [
            max(ans.is_correct * self.time_effect(diff), 0) for ans, diff
            in izip(item.practices, item.get_diffs(question.inserted))
        ]
        incorrect_weights = [
            (1 - ans.is_correct) * self.time_effect(diff) for ans, diff
            in izip(item.practices, item.get_diffs(question.inserted))
        ]
        return sum(correct_weights), sum(incorrect_weights)

class PFATiming(PFAGong):
    """Performance Factor Analysis combining some aspects of both
    the Yue Gong's PFA and the ACT-R model.

    :param gamma: The significance of the update when the student
        answers correctly.
    :type gamma: float
    :param delta: The significance of the update when the student
        answers incorrectly.
    :type delta: float
    :param time_effect_good: Time effect function for correct answers.
    :type time_effect_good: callable or string
    :param time_effect_bad: Time effect function for wrong answers.
    :type time_effect_bad: callable or string
    """
    ABBR = 'PFA/G/T'

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('gamma', 1)  # these parameters should not be
        kwargs.setdefault('delta', 1)  # modified, i.e. kept equal to 1

        time_effect_good = kwargs.pop('time_effect_good', 'pow')
        time_effect_bad = kwargs.pop('time_effect_bad', 'pow')

        if isinstance(time_effect_good, basestring):
            self.a, self.c = kwargs.pop('a', None), kwargs.pop('c', None)
            self.time_effect_good = init_time_effect(
                self, time_effect_good, parameters=('a', 'c'))
        else:
            self.time_effect_good = time_effect_good

        if isinstance(time_effect_bad, basestring):
            self.b, self.d = kwargs.pop('b', None), kwargs.pop('d', None)
            self.time_effect_bad = init_time_effect(
                self, time_effect_bad, parameters=('b', 'd'))
        else:
            self.time_effect_bad = time_effect_bad

        super(PFAGong, self).__init__(*args, **kwargs)

    def get_weights(self, item, question):
        """Returns weights of previous answers to the given item.

        :param item: *Item* (i.e. practiced place by a user).
        :type item: :class:`Item`
        :param question: Asked question.
        :type question: :class:`pandas.Series` or :class:`Question`
        """
        correct_weights = [
            ans.is_correct * self.time_effect_good(diff) for ans, diff
            in izip(item.practices, item.get_diffs(question.inserted))
        ]
        incorrect_weights = [
            (1 - ans.is_correct) * self.time_effect_bad(diff) for ans, diff
            in izip(item.practices, item.get_diffs(question.inserted))
        ]
        return sum(correct_weights), sum(incorrect_weights)