import pandas as pd
import numpy as np
from datetime import datetime
from pvlib.bifacial import pvfactors_timeseries, PVFactorsReportBuilder
from conftest import requires_pvfactors
import pytest

@requires_pvfactors
@pytest.mark.parametrize('run_parallel_calculations',
                         [False, True])
def test_pvfactors_timeseries(run_parallel_calculations):
    """ Test that pvfactors is functional, using the TLDR section inputs of the
    package github repo README.md file:
    https://github.com/SunPower/pvfactors/blob/master/README.md#tldr---quick-start"""

    # Create some inputs
    timestamps = pd.DatetimeIndex([datetime(2017, 8, 31, 11),
                                   datetime(2017, 8, 31, 12)]
                                  ).set_names('timestamps')
    solar_zenith = [20., 10.]
    solar_azimuth = [110., 140.]
    surface_tilt = [10., 0.]
    surface_azimuth = [90., 90.]
    axis_azimuth = 0.
    dni = [1000., 300.]
    dhi = [50., 500.]
    gcr = 0.4
    pvrow_height = 1.75
    pvrow_width = 2.44
    albedo = 0.2
    n_pvrows = 3
    index_observed_pvrow = 1
    rho_front_pvrow = 0.03
    rho_back_pvrow = 0.05
    horizon_band_angle = 15.

    # Expected values
    expected_ipoa_front = pd.Series([1034.95474708997, 795.4423259036623],
                                    index=timestamps,
                                    name=('total_inc_front'))
    expected_ipoa_back = pd.Series([91.88707460262768, 78.05831585685215],
                                   index=timestamps,
                                   name=('total_inc_back'))

    # Run calculation
    ipoa_front, ipoa_back = pvfactors_timeseries(
        solar_azimuth, solar_zenith, surface_azimuth, surface_tilt,
        axis_azimuth,
        timestamps, dni, dhi, gcr, pvrow_height, pvrow_width, albedo,
        n_pvrows=n_pvrows, index_observed_pvrow=index_observed_pvrow,
        rho_front_pvrow=rho_front_pvrow, rho_back_pvrow=rho_back_pvrow,
        horizon_band_angle=horizon_band_angle,
        run_parallel_calculations=run_parallel_calculations,
        n_workers_for_parallel_calcs=-1)

    pd.testing.assert_series_equal(ipoa_front, expected_ipoa_front)
    pd.testing.assert_series_equal(ipoa_back, expected_ipoa_back)

@requires_pvfactors
@pytest.mark.parametrize('run_parallel_calculations',
                         [False, True])
def test_pvfactors_timeseries_pandas_inputs(run_parallel_calculations):
    """ Test that pvfactors is functional, using the TLDR section inputs of the
    package github repo README.md file, but converted to pandas Series:
    https://github.com/SunPower/pvfactors/blob/master/README.md#tldr---quick-start"""

    # Create some inputs
    timestamps = pd.DatetimeIndex([datetime(2017, 8, 31, 11),
                                   datetime(2017, 8, 31, 12)]
                                  ).set_names('timestamps')
    solar_zenith = pd.Series([20., 10.])
    solar_azimuth = pd.Series([110., 140.])
    surface_tilt = pd.Series([10., 0.])
    surface_azimuth = pd.Series([90., 90.])
    axis_azimuth = 0.
    dni = pd.Series([1000., 300.])
    dhi = pd.Series([50., 500.])
    gcr = 0.4
    pvrow_height = 1.75
    pvrow_width = 2.44
    albedo = 0.2
    n_pvrows = 3
    index_observed_pvrow = 1
    rho_front_pvrow = 0.03
    rho_back_pvrow = 0.05
    horizon_band_angle = 15.

    # Expected values
    expected_ipoa_front = pd.Series([1034.95474708997, 795.4423259036623],
                                    index=timestamps,
                                    name=('total_inc_front'))
    expected_ipoa_back = pd.Series([91.88707460262768, 78.05831585685215],
                                   index=timestamps,
                                   name=('total_inc_back'))

    # Run calculation
    ipoa_front, ipoa_back = pvfactors_timeseries(
        solar_azimuth, solar_zenith, surface_azimuth, surface_tilt,
        axis_azimuth,
        timestamps, dni, dhi, gcr, pvrow_height, pvrow_width, albedo,
        n_pvrows=n_pvrows, index_observed_pvrow=index_observed_pvrow,
        rho_front_pvrow=rho_front_pvrow, rho_back_pvrow=rho_back_pvrow,
        horizon_band_angle=horizon_band_angle,
        run_parallel_calculations=run_parallel_calculations,
        n_workers_for_parallel_calcs=-1)

    pd.testing.assert_series_equal(ipoa_front, expected_ipoa_front)
    pd.testing.assert_series_equal(ipoa_back, expected_ipoa_back)

def test_build_1():
    """Test that build correctly instantiates a dictionary, when passed a Nones
    for the report and pvarray arguments.
    """
    report = None
    pvarray = None
    expected = {'total_inc_back': [np.nan], 'total_inc_front': [np.nan]}
    assert expected == PVFactorsReportBuilder.build(report, pvarray)

def test_merge_1():
    """Test that merge correctly returns the first element of the reports
    argument when there is only dictionary in reports.
    """
    test_dict = {'total_inc_back': [1, 2, 3], 'total_inc_front': [4, 5, 6]}
    reports = [test_dict]
    assert test_dict == PVFactorsReportBuilder.merge(reports)

def test_merge_2():
    """Test that merge correctly combines two dictionary reports.
    """
    test_dict_1 = {'total_inc_back': [1, 2], 'total_inc_front': [4, 5]}
    test_dict_2 = {'total_inc_back': [3], 'total_inc_front': [6]}
    expected = {'total_inc_back': [1, 2, 3], 'total_inc_front': [4, 5, 6]}
    reports = [test_dict_1, test_dict_2]
    assert expected == PVFactorsReportBuilder.merge(reports)

def test_merge_3():
    """Test that merge correctly combines three dictionary reports.
    """
    test_dict_1 = {'total_inc_back': [1], 'total_inc_front': [4]}
    test_dict_2 = {'total_inc_back': [2], 'total_inc_front': [5]}
    test_dict_3 = {'total_inc_back': [3], 'total_inc_front': [6]}
    expected = {'total_inc_back': [1, 2, 3], 'total_inc_front': [4, 5, 6]}
    reports = [test_dict_1, test_dict_2, test_dict_3]
    assert expected == PVFactorsReportBuilder.merge(reports)


"""
Benchmark scikit-learn's Ward implement compared to SciPy's
"""

import time

import numpy as np
from scipy.cluster import hierarchy
import pylab as pl

from sklearn.cluster import AgglomerativeClustering

ward = AgglomerativeClustering(n_clusters=3, linkage='ward')

n_samples = np.logspace(.5, 3, 9)
n_features = np.logspace(1, 3.5, 7)
N_samples, N_features = np.meshgrid(n_samples,
                                    n_features)
scikits_time = np.zeros(N_samples.shape)
scipy_time = np.zeros(N_samples.shape)

for i, n in enumerate(n_samples):
    for j, p in enumerate(n_features):
        X = np.random.normal(size=(n, p))
        t0 = time.time()
        ward.fit(X)
        scikits_time[j, i] = time.time() - t0
        t0 = time.time()
        hierarchy.ward(X)
        scipy_time[j, i] = time.time() - t0

ratio = scikits_time / scipy_time

pl.figure("scikit-learn Ward's method benchmark results")
pl.imshow(np.log(ratio), aspect='auto', origin="lower")
pl.colorbar()
pl.contour(ratio, levels=[1, ], colors='k')
pl.yticks(range(len(n_features)), n_features.astype(np.int))
pl.ylabel('N features')
pl.xticks(range(len(n_samples)), n_samples.astype(np.int))
pl.xlabel('N samples')
pl.title("Scikit's time, in units of scipy time (log)")
pl.show()


import pytest

import numpy as np
import pandas as pd
from pandas import (DataFrame, Index, compat, isna,
                    Series, MultiIndex, Timestamp, date_range)
from pandas.errors import UnsupportedFunctionCall
from pandas.util import testing as tm
import pandas.core.nanops as nanops
from string import ascii_lowercase
from pandas.compat import product as cart_product

@pytest.mark.parametrize("agg_func", ['any', 'all'])
@pytest.mark.parametrize("skipna", [True, False])
@pytest.mark.parametrize("vals", [
    ['foo', 'bar', 'baz'], ['foo', '', ''], ['', '', ''],
    [1, 2, 3], [1, 0, 0], [0, 0, 0],
    [1., 2., 3.], [1., 0., 0.], [0., 0., 0.],
    [True, True, True], [True, False, False], [False, False, False],
    [np.nan, np.nan, np.nan]
])
def test_groupby_bool_aggs(agg_func, skipna, vals):
    df = DataFrame({'key': ['a'] * 3 + ['b'] * 3, 'val': vals * 2})

    # Figure out expectation using Python builtin
    exp = getattr(compat.builtins, agg_func)(vals)

    # edge case for missing data with skipna and 'any'
    if skipna and all(isna(vals)) and agg_func == 'any':
        exp = False

    exp_df = DataFrame([exp] * 2, columns=['val'], index=Index(
        ['a', 'b'], name='key'))
    result = getattr(df.groupby('key'), agg_func)(skipna=skipna)
    tm.assert_frame_equal(result, exp_df)

def test_max_min_non_numeric():
    # #2700
    aa = DataFrame({'nn': [11, 11, 22, 22],
                    'ii': [1, 2, 3, 4],
                    'ss': 4 * ['mama']})

    result = aa.groupby('nn').max()
    assert 'ss' in result

    result = aa.groupby('nn').max(numeric_only=False)
    assert 'ss' in result

    result = aa.groupby('nn').min()
    assert 'ss' in result

    result = aa.groupby('nn').min(numeric_only=False)
    assert 'ss' in result

def test_intercept_builtin_sum():
    s = Series([1., 2., np.nan, 3.])
    grouped = s.groupby([0, 1, 2, 2])

    result = grouped.agg(compat.builtins.sum)
    result2 = grouped.apply(compat.builtins.sum)
    expected = grouped.sum()
    tm.assert_series_equal(result, expected)
    tm.assert_series_equal(result2, expected)

# @pytest.mark.parametrize("f", [max, min, sum])
# def test_builtins_apply(f):

@pytest.mark.parametrize("f", [max, min, sum])
@pytest.mark.parametrize('keys', [
    "jim",  # Single key
    ["jim", "joe"]  # Multi-key
])
def test_builtins_apply(keys, f):
    # see gh-8155
    df = pd.DataFrame(np.random.randint(1, 50, (1000, 2)),
                      columns=["jim", "joe"])
    df["jolie"] = np.random.randn(1000)

    fname = f.__name__
    result = df.groupby(keys).apply(f)
    ngroups = len(df.drop_duplicates(subset=keys))

    assert_msg = ("invalid frame shape: {} "
                  "(expected ({}, 3))".format(result.shape, ngroups))
    assert result.shape == (ngroups, 3), assert_msg

    tm.assert_frame_equal(result,  # numpy's equivalent function
                          df.groupby(keys).apply(getattr(np, fname)))

    if f != sum:
        expected = df.groupby(keys).agg(fname).reset_index()
        expected.set_index(keys, inplace=True, drop=False)
        tm.assert_frame_equal(result, expected, check_dtype=False)

    tm.assert_series_equal(getattr(result, fname)(),
                           getattr(df, fname)())

def test_arg_passthru():
    # make sure that we are passing thru kwargs
    # to our agg functions

    # GH3668
    # GH5724
    df = pd.DataFrame(
        {'group': [1, 1, 2],
         'int': [1, 2, 3],
         'float': [4., 5., 6.],
         'string': list('abc'),
         'category_string': pd.Series(list('abc')).astype('category'),
         'category_int': [7, 8, 9],
         'datetime': pd.date_range('20130101', periods=3),
         'datetimetz': pd.date_range('20130101',
                                     periods=3,
                                     tz='US/Eastern'),
         'timedelta': pd.timedelta_range('1 s', periods=3, freq='s')},
        columns=['group', 'int', 'float', 'string',
                 'category_string', 'category_int',
                 'datetime', 'datetimetz',
                 'timedelta'])

    expected_columns_numeric = Index(['int', 'float', 'category_int'])

    # mean / median
    expected = pd.DataFrame(
        {'category_int': [7.5, 9],
         'float': [4.5, 6.],
         'timedelta': [pd.Timedelta('1.5s'),
                       pd.Timedelta('3s')],
         'int': [1.5, 3],
         'datetime': [pd.Timestamp('2013-01-01 12:00:00'),
                      pd.Timestamp('2013-01-03 00:00:00')],
         'datetimetz': [
             pd.Timestamp('2013-01-01 12:00:00', tz='US/Eastern'),
             pd.Timestamp('2013-01-03 00:00:00', tz='US/Eastern')]},
        index=Index([1, 2], name='group'),
        columns=['int', 'float', 'category_int',
                 'datetime', 'datetimetz', 'timedelta'])
    for attr in ['mean', 'median']:
        f = getattr(df.groupby('group'), attr)
        result = f()
        tm.assert_index_equal(result.columns, expected_columns_numeric)

        result = f(numeric_only=False)
        tm.assert_frame_equal(result.reindex_like(expected), expected)

    # TODO: min, max *should* handle
    # categorical (ordered) dtype
    expected_columns = Index(['int', 'float', 'string',
                              'category_int',
                              'datetime', 'datetimetz',
                              'timedelta'])
    for attr in ['min', 'max']:
        f = getattr(df.groupby('group'), attr)
        result = f()
        tm.assert_index_equal(result.columns, expected_columns)

        result = f(numeric_only=False)
        tm.assert_index_equal(result.columns, expected_columns)

    expected_columns = Index(['int', 'float', 'string',
                              'category_string', 'category_int',
                              'datetime', 'datetimetz',
                              'timedelta'])
    for attr in ['first', 'last']:
        f = getattr(df.groupby('group'), attr)
        result = f()
        tm.assert_index_equal(result.columns, expected_columns)

        result = f(numeric_only=False)
        tm.assert_index_equal(result.columns, expected_columns)

    expected_columns = Index(['int', 'float', 'string',
                              'category_int', 'timedelta'])
    for attr in ['sum']:
        f = getattr(df.groupby('group'), attr)
        result = f()
        tm.assert_index_equal(result.columns, expected_columns_numeric)

        result = f(numeric_only=False)
        tm.assert_index_equal(result.columns, expected_columns)

    expected_columns = Index(['int', 'float', 'category_int'])
    for attr in ['prod', 'cumprod']:
        f = getattr(df.groupby('group'), attr)
        result = f()
        tm.assert_index_equal(result.columns, expected_columns_numeric)

        result = f(numeric_only=False)
        tm.assert_index_equal(result.columns, expected_columns)

    # like min, max, but don't include strings
    expected_columns = Index(['int', 'float',
                              'category_int',
                              'datetime', 'datetimetz',
                              'timedelta'])
    for attr in ['cummin', 'cummax']:
        f = getattr(df.groupby('group'), attr)
        result = f()
        # GH 15561: numeric_only=False set by default like min/max
        tm.assert_index_equal(result.columns, expected_columns)

        result = f(numeric_only=False)
        tm.assert_index_equal(result.columns, expected_columns)

    expected_columns = Index(['int', 'float', 'category_int',
                              'timedelta'])
    for attr in ['cumsum']:
        f = getattr(df.groupby('group'), attr)
        result = f()
        tm.assert_index_equal(result.columns, expected_columns_numeric)

        result = f(numeric_only=False)
        tm.assert_index_equal(result.columns, expected_columns)

def test_non_cython_api():

    # GH5610
    # non-cython calls should not include the grouper

    df = DataFrame(
        [[1, 2, 'foo'],
         [1, np.nan, 'bar'],
         [3, np.nan, 'baz']],
        columns=['A', 'B', 'C'])
    g = df.groupby('A')
    gni = df.groupby('A', as_index=False)

    # mad
    expected = DataFrame([[0], [np.nan]], columns=['B'], index=[1, 3])
    expected.index.name = 'A'
    result = g.mad()
    tm.assert_frame_equal(result, expected)

    expected = DataFrame([[0., 0.], [0, np.nan]], columns=['A', 'B'],
                         index=[0, 1])
    result = gni.mad()
    tm.assert_frame_equal(result, expected)

    # describe
    expected_index = pd.Index([1, 3], name='A')
    expected_col = pd.MultiIndex(levels=[['B'],
                                         ['count', 'mean', 'std', 'min',
                                          '25%', '50%', '75%', 'max']],
                                 labels=[[0] * 8, list(range(8))])
    expected = pd.DataFrame([[1.0, 2.0, np.nan, 2.0, 2.0, 2.0, 2.0, 2.0],
                             [0.0, np.nan, np.nan, np.nan, np.nan, np.nan,
                              np.nan, np.nan]],
                            index=expected_index,
                            columns=expected_col)
    result = g.describe()
    tm.assert_frame_equal(result, expected)

    expected = pd.concat([df[df.A == 1].describe().unstack().to_frame().T,
                          df[df.A == 3].describe().unstack().to_frame().T])
    expected.index = pd.Index([0, 1])
    result = gni.describe()
    tm.assert_frame_equal(result, expected)

    # any
    expected = DataFrame([[True, True], [False, True]], columns=['B', 'C'],
                         index=[1, 3])
    expected.index.name = 'A'
    result = g.any()
    tm.assert_frame_equal(result, expected)

    # idxmax
    expected = DataFrame([[0.0], [np.nan]], columns=['B'], index=[1, 3])
    expected.index.name = 'A'
    result = g.idxmax()
    tm.assert_frame_equal(result, expected)

def test_cython_api2():

    # this takes the fast apply path

    # cumsum (GH5614)
    df = DataFrame(
        [[1, 2, np.nan], [1, np.nan, 9], [3, 4, 9]
         ], columns=['A', 'B', 'C'])
    expected = DataFrame(
        [[2, np.nan], [np.nan, 9], [4, 9]], columns=['B', 'C'])
    result = df.groupby('A').cumsum()
    tm.assert_frame_equal(result, expected)

    # GH 5755 - cumsum is a transformer and should ignore as_index
    result = df.groupby('A', as_index=False).cumsum()
    tm.assert_frame_equal(result, expected)

    # GH 13994
    result = df.groupby('A').cumsum(axis=1)
    expected = df.cumsum(axis=1)
    tm.assert_frame_equal(result, expected)
    result = df.groupby('A').cumprod(axis=1)
    expected = df.cumprod(axis=1)
    tm.assert_frame_equal(result, expected)

def test_cython_median():
    df = DataFrame(np.random.randn(1000))
    df.values[::2] = np.nan

    labels = np.random.randint(0, 50, size=1000).astype(float)
    labels[::17] = np.nan

    result = df.groupby(labels).median()
    exp = df.groupby(labels).agg(nanops.nanmedian)
    tm.assert_frame_equal(result, exp)

    df = DataFrame(np.random.randn(1000, 5))
    rs = df.groupby(labels).agg(np.median)
    xp = df.groupby(labels).median()
    tm.assert_frame_equal(rs, xp)

def test_median_empty_bins(observed):
    df = pd.DataFrame(np.random.randint(0, 44, 500))

    grps = range(0, 55, 5)
    bins = pd.cut(df[0], grps)

    result = df.groupby(bins, observed=observed).median()
    expected = df.groupby(bins, observed=observed).agg(lambda x: x.median())
    tm.assert_frame_equal(result, expected)

@pytest.mark.parametrize("dtype", [
    'int8', 'int16', 'int32', 'int64', 'float32', 'float64'])
@pytest.mark.parametrize("method,data", [
    ('first', {'df': [{'a': 1, 'b': 1}, {'a': 2, 'b': 3}]}),
    ('last', {'df': [{'a': 1, 'b': 2}, {'a': 2, 'b': 4}]}),
    ('min', {'df': [{'a': 1, 'b': 1}, {'a': 2, 'b': 3}]}),
    ('max', {'df': [{'a': 1, 'b': 2}, {'a': 2, 'b': 4}]}),
    ('nth', {'df': [{'a': 1, 'b': 2}, {'a': 2, 'b': 4}],
             'args': [1]}),
    ('count', {'df': [{'a': 1, 'b': 2}, {'a': 2, 'b': 2}],
               'out_type': 'int64'})
])
def test_groupby_non_arithmetic_agg_types(dtype, method, data):
    # GH9311, GH6620
    df = pd.DataFrame(
        [{'a': 1, 'b': 1},
         {'a': 1, 'b': 2},
         {'a': 2, 'b': 3},
         {'a': 2, 'b': 4}])

    df['b'] = df.b.astype(dtype)

    if 'args' not in data:
        data['args'] = []

    if 'out_type' in data:
        out_type = data['out_type']
    else:
        out_type = dtype

    exp = data['df']
    df_out = pd.DataFrame(exp)

    df_out['b'] = df_out.b.astype(out_type)
    df_out.set_index('a', inplace=True)

    grpd = df.groupby('a')
    t = getattr(grpd, method)(*data['args'])
    tm.assert_frame_equal(t, df_out)

@pytest.mark.parametrize("i", [
    (Timestamp("2011-01-15 12:50:28.502376"),
     Timestamp("2011-01-20 12:50:28.593448")),
    (24650000000000001, 24650000000000002)
])
def test_groupby_non_arithmetic_agg_int_like_precision(i):
    # see gh-6620, gh-9311
    df = pd.DataFrame([{"a": 1, "b": i[0]}, {"a": 1, "b": i[1]}])

    grp_exp = {"first": {"expected": i[0]},
               "last": {"expected": i[1]},
               "min": {"expected": i[0]},
               "max": {"expected": i[1]},
               "nth": {"expected": i[1],
                       "args": [1]},
               "count": {"expected": 2}}

    for method, data in compat.iteritems(grp_exp):
        if "args" not in data:
            data["args"] = []

        grouped = df.groupby("a")
        res = getattr(grouped, method)(*data["args"])

        assert res.iloc[0].b == data["expected"]

def test_fill_consistency():

    # GH9221
    # pass thru keyword arguments to the generated wrapper
    # are set if the passed kw is None (only)
    df = DataFrame(index=pd.MultiIndex.from_product(
        [['value1', 'value2'], date_range('2014-01-01', '2014-01-06')]),
        columns=Index(
        ['1', '2'], name='id'))
    df['1'] = [np.nan, 1, np.nan, np.nan, 11, np.nan, np.nan, 2, np.nan,
               np.nan, 22, np.nan]
    df['2'] = [np.nan, 3, np.nan, np.nan, 33, np.nan, np.nan, 4, np.nan,
               np.nan, 44, np.nan]

    expected = df.groupby(level=0, axis=0).fillna(method='ffill')
    result = df.T.groupby(level=0, axis=1).fillna(method='ffill').T
    tm.assert_frame_equal(result, expected)

def test_groupby_cumprod():
    # GH 4095
    df = pd.DataFrame({'key': ['b'] * 10, 'value': 2})

    actual = df.groupby('key')['value'].cumprod()
    expected = df.groupby('key')['value'].apply(lambda x: x.cumprod())
    expected.name = 'value'
    tm.assert_series_equal(actual, expected)

    df = pd.DataFrame({'key': ['b'] * 100, 'value': 2})
    actual = df.groupby('key')['value'].cumprod()
    # if overflows, groupby product casts to float
    # while numpy passes back invalid values
    df['value'] = df['value'].astype(float)
    expected = df.groupby('key')['value'].apply(lambda x: x.cumprod())
    expected.name = 'value'
    tm.assert_series_equal(actual, expected)

def test_ops_general():
    ops = [('mean', np.mean),
           ('median', np.median),
           ('std', np.std),
           ('var', np.var),
           ('sum', np.sum),
           ('prod', np.prod),
           ('min', np.min),
           ('max', np.max),
           ('first', lambda x: x.iloc[0]),
           ('last', lambda x: x.iloc[-1]),
           ('count', np.size), ]
    try:
        from scipy.stats import sem
    except ImportError:
        pass
    else:
        ops.append(('sem', sem))
    df = DataFrame(np.random.randn(1000))
    labels = np.random.randint(0, 50, size=1000).astype(float)

    for op, targop in ops:
        result = getattr(df.groupby(labels), op)().astype(float)
        expected = df.groupby(labels).agg(targop)
        try:
            tm.assert_frame_equal(result, expected)
        except BaseException as exc:
            exc.args += ('operation: %s' % op, )
            raise

def test_max_nan_bug():
    raw = """,Date,app,File
-04-23,2013-04-23 00:00:00,,log080001.log
-05-06,2013-05-06 00:00:00,,log.log
-05-07,2013-05-07 00:00:00,OE,xlsx"""

    df = pd.read_csv(compat.StringIO(raw), parse_dates=[0])
    gb = df.groupby('Date')
    r = gb[['File']].max()
    e = gb['File'].max().to_frame()
    tm.assert_frame_equal(r, e)
    assert not r['File'].isna().any()

def test_nlargest():
    a = Series([1, 3, 5, 7, 2, 9, 0, 4, 6, 10])
    b = Series(list('a' * 5 + 'b' * 5))
    gb = a.groupby(b)
    r = gb.nlargest(3)
    e = Series([
        7, 5, 3, 10, 9, 6
    ], index=MultiIndex.from_arrays([list('aaabbb'), [3, 2, 1, 9, 5, 8]]))
    tm.assert_series_equal(r, e)

    a = Series([1, 1, 3, 2, 0, 3, 3, 2, 1, 0])
    gb = a.groupby(b)
    e = Series([
        3, 2, 1, 3, 3, 2
    ], index=MultiIndex.from_arrays([list('aaabbb'), [2, 3, 1, 6, 5, 7]]))
    tm.assert_series_equal(gb.nlargest(3, keep='last'), e)

def test_nsmallest():
    a = Series([1, 3, 5, 7, 2, 9, 0, 4, 6, 10])
    b = Series(list('a' * 5 + 'b' * 5))
    gb = a.groupby(b)
    r = gb.nsmallest(3)
    e = Series([
        1, 2, 3, 0, 4, 6
    ], index=MultiIndex.from_arrays([list('aaabbb'), [0, 4, 1, 6, 7, 8]]))
    tm.assert_series_equal(r, e)

    a = Series([1, 1, 3, 2, 0, 3, 3, 2, 1, 0])
    gb = a.groupby(b)
    e = Series([
        0, 1, 1, 0, 1, 2
    ], index=MultiIndex.from_arrays([list('aaabbb'), [4, 1, 0, 9, 8, 7]]))
    tm.assert_series_equal(gb.nsmallest(3, keep='last'), e)

def test_numpy_compat():
    # see gh-12811
    df = pd.DataFrame({'A': [1, 2, 1], 'B': [1, 2, 3]})
    g = df.groupby('A')

    msg = "numpy operations are not valid with groupby"

    for func in ('mean', 'var', 'std', 'cumprod', 'cumsum'):
        tm.assert_raises_regex(UnsupportedFunctionCall, msg,
                               getattr(g, func), 1, 2, 3)
        tm.assert_raises_regex(UnsupportedFunctionCall, msg,
                               getattr(g, func), foo=1)

def test_cummin_cummax():
    # GH 15048
    num_types = [np.int32, np.int64, np.float32, np.float64]
    num_mins = [np.iinfo(np.int32).min, np.iinfo(np.int64).min,
                np.finfo(np.float32).min, np.finfo(np.float64).min]
    num_max = [np.iinfo(np.int32).max, np.iinfo(np.int64).max,
               np.finfo(np.float32).max, np.finfo(np.float64).max]
    base_df = pd.DataFrame({'A': [1, 1, 1, 1, 2, 2, 2, 2],
                            'B': [3, 4, 3, 2, 2, 3, 2, 1]})
    expected_mins = [3, 3, 3, 2, 2, 2, 2, 1]
    expected_maxs = [3, 4, 4, 4, 2, 3, 3, 3]

    for dtype, min_val, max_val in zip(num_types, num_mins, num_max):
        df = base_df.astype(dtype)

        # cummin
        expected = pd.DataFrame({'B': expected_mins}).astype(dtype)
        result = df.groupby('A').cummin()
        tm.assert_frame_equal(result, expected)
        result = df.groupby('A').B.apply(lambda x: x.cummin()).to_frame()
        tm.assert_frame_equal(result, expected)

        # Test cummin w/ min value for dtype
        df.loc[[2, 6], 'B'] = min_val
        expected.loc[[2, 3, 6, 7], 'B'] = min_val
        result = df.groupby('A').cummin()
        tm.assert_frame_equal(result, expected)
        expected = df.groupby('A').B.apply(lambda x: x.cummin()).to_frame()
        tm.assert_frame_equal(result, expected)

        # cummax
        expected = pd.DataFrame({'B': expected_maxs}).astype(dtype)
        result = df.groupby('A').cummax()
        tm.assert_frame_equal(result, expected)
        result = df.groupby('A').B.apply(lambda x: x.cummax()).to_frame()
        tm.assert_frame_equal(result, expected)

        # Test cummax w/ max value for dtype
        df.loc[[2, 6], 'B'] = max_val
        expected.loc[[2, 3, 6, 7], 'B'] = max_val
        result = df.groupby('A').cummax()
        tm.assert_frame_equal(result, expected)
        expected = df.groupby('A').B.apply(lambda x: x.cummax()).to_frame()
        tm.assert_frame_equal(result, expected)

    # Test nan in some values
    base_df.loc[[0, 2, 4, 6], 'B'] = np.nan
    expected = pd.DataFrame({'B': [np.nan, 4, np.nan, 2,
                                   np.nan, 3, np.nan, 1]})
    result = base_df.groupby('A').cummin()
    tm.assert_frame_equal(result, expected)
    expected = (base_df.groupby('A')
                       .B
                       .apply(lambda x: x.cummin())
                       .to_frame())
    tm.assert_frame_equal(result, expected)

    expected = pd.DataFrame({'B': [np.nan, 4, np.nan, 4,
                                   np.nan, 3, np.nan, 3]})
    result = base_df.groupby('A').cummax()
    tm.assert_frame_equal(result, expected)
    expected = (base_df.groupby('A')
                       .B
                       .apply(lambda x: x.cummax())
                       .to_frame())
    tm.assert_frame_equal(result, expected)

    # Test nan in entire column
    base_df['B'] = np.nan
    expected = pd.DataFrame({'B': [np.nan] * 8})
    result = base_df.groupby('A').cummin()
    tm.assert_frame_equal(expected, result)
    result = base_df.groupby('A').B.apply(lambda x: x.cummin()).to_frame()
    tm.assert_frame_equal(expected, result)
    result = base_df.groupby('A').cummax()
    tm.assert_frame_equal(expected, result)
    result = base_df.groupby('A').B.apply(lambda x: x.cummax()).to_frame()
    tm.assert_frame_equal(expected, result)

    # GH 15561
    df = pd.DataFrame(dict(a=[1], b=pd.to_datetime(['2001'])))
    expected = pd.Series(pd.to_datetime('2001'), index=[0], name='b')
    for method in ['cummax', 'cummin']:
        result = getattr(df.groupby('a')['b'], method)()
        tm.assert_series_equal(expected, result)

    # GH 15635
    df = pd.DataFrame(dict(a=[1, 2, 1], b=[2, 1, 1]))
    result = df.groupby('a').b.cummax()
    expected = pd.Series([2, 1, 2], name='b')
    tm.assert_series_equal(result, expected)

    df = pd.DataFrame(dict(a=[1, 2, 1], b=[1, 2, 2]))
    result = df.groupby('a').b.cummin()
    expected = pd.Series([1, 2, 1], name='b')
    tm.assert_series_equal(result, expected)

@pytest.mark.parametrize('in_vals, out_vals', [

    # Basics: strictly increasing (T), strictly decreasing (F),
    # abs val increasing (F), non-strictly increasing (T)
    ([1, 2, 5, 3, 2, 0, 4, 5, -6, 1, 1],
     [True, False, False, True]),

    # Test with inf vals
    ([1, 2.1, np.inf, 3, 2, np.inf, -np.inf, 5, 11, 1, -np.inf],
     [True, False, True, False]),

    # Test with nan vals; should always be False
    ([1, 2, np.nan, 3, 2, np.nan, np.nan, 5, -np.inf, 1, np.nan],
     [False, False, False, False]),
])
def test_is_monotonic_increasing(in_vals, out_vals):
    # GH 17015
    source_dict = {
        'A': ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11'],
        'B': ['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'c', 'd', 'd'],
        'C': in_vals}
    df = pd.DataFrame(source_dict)
    result = df.groupby('B').C.is_monotonic_increasing
    index = Index(list('abcd'), name='B')
    expected = pd.Series(index=index, data=out_vals, name='C')
    tm.assert_series_equal(result, expected)

    # Also check result equal to manually taking x.is_monotonic_increasing.
    expected = (
        df.groupby(['B']).C.apply(lambda x: x.is_monotonic_increasing))
    tm.assert_series_equal(result, expected)

@pytest.mark.parametrize('in_vals, out_vals', [
    # Basics: strictly decreasing (T), strictly increasing (F),
    # abs val decreasing (F), non-strictly increasing (T)
    ([10, 9, 7, 3, 4, 5, -3, 2, 0, 1, 1],
     [True, False, False, True]),

    # Test with inf vals
    ([np.inf, 1, -np.inf, np.inf, 2, -3, -np.inf, 5, -3, -np.inf, -np.inf],
     [True, True, False, True]),

    # Test with nan vals; should always be False
    ([1, 2, np.nan, 3, 2, np.nan, np.nan, 5, -np.inf, 1, np.nan],
     [False, False, False, False]),
])
def test_is_monotonic_decreasing(in_vals, out_vals):
    # GH 17015
    source_dict = {
        'A': ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11'],
        'B': ['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'c', 'd', 'd'],
        'C': in_vals}

    df = pd.DataFrame(source_dict)
    result = df.groupby('B').C.is_monotonic_decreasing
    index = Index(list('abcd'), name='B')
    expected = pd.Series(index=index, data=out_vals, name='C')
    tm.assert_series_equal(result, expected)

# describe
# --------------------------------

def test_apply_describe_bug(mframe):
    grouped = mframe.groupby(level='first')
    grouped.describe()  # it works!

def test_series_describe_multikey():
    ts = tm.makeTimeSeries()
    grouped = ts.groupby([lambda x: x.year, lambda x: x.month])
    result = grouped.describe()
    tm.assert_series_equal(result['mean'], grouped.mean(),
                           check_names=False)
    tm.assert_series_equal(result['std'], grouped.std(), check_names=False)
    tm.assert_series_equal(result['min'], grouped.min(), check_names=False)

def test_series_describe_single():
    ts = tm.makeTimeSeries()
    grouped = ts.groupby(lambda x: x.month)
    result = grouped.apply(lambda x: x.describe())
    expected = grouped.describe().stack()
    tm.assert_series_equal(result, expected)

def test_series_index_name(df):
    grouped = df.loc[:, ['C']].groupby(df['A'])
    result = grouped.agg(lambda x: x.mean())
    assert result.index.name == 'A'

def test_frame_describe_multikey(tsframe):
    grouped = tsframe.groupby([lambda x: x.year, lambda x: x.month])
    result = grouped.describe()
    desc_groups = []
    for col in tsframe:
        group = grouped[col].describe()
        # GH 17464 - Remove duplicate MultiIndex levels
        group_col = pd.MultiIndex(
            levels=[[col], group.columns],
            labels=[[0] * len(group.columns), range(len(group.columns))])
        group = pd.DataFrame(group.values,
                             columns=group_col,
                             index=group.index)
        desc_groups.append(group)
    expected = pd.concat(desc_groups, axis=1)
    tm.assert_frame_equal(result, expected)

    groupedT = tsframe.groupby({'A': 0, 'B': 0,
                                'C': 1, 'D': 1}, axis=1)
    result = groupedT.describe()
    expected = tsframe.describe().T
    expected.index = pd.MultiIndex(
        levels=[[0, 1], expected.index],
        labels=[[0, 0, 1, 1], range(len(expected.index))])
    tm.assert_frame_equal(result, expected)

def test_frame_describe_tupleindex():

    # GH 14848 - regression from 0.19.0 to 0.19.1
    df1 = DataFrame({'x': [1, 2, 3, 4, 5] * 3,
                     'y': [10, 20, 30, 40, 50] * 3,
                     'z': [100, 200, 300, 400, 500] * 3})
    df1['k'] = [(0, 0, 1), (0, 1, 0), (1, 0, 0)] * 5
    df2 = df1.rename(columns={'k': 'key'})
    pytest.raises(ValueError, lambda: df1.groupby('k').describe())
    pytest.raises(ValueError, lambda: df2.groupby('key').describe())

def test_frame_describe_unstacked_format():
    # GH 4792
    prices = {pd.Timestamp('2011-01-06 10:59:05', tz=None): 24990,
              pd.Timestamp('2011-01-06 12:43:33', tz=None): 25499,
              pd.Timestamp('2011-01-06 12:54:09', tz=None): 25499}
    volumes = {pd.Timestamp('2011-01-06 10:59:05', tz=None): 1500000000,
               pd.Timestamp('2011-01-06 12:43:33', tz=None): 5000000000,
               pd.Timestamp('2011-01-06 12:54:09', tz=None): 100000000}
    df = pd.DataFrame({'PRICE': prices,
                       'VOLUME': volumes})
    result = df.groupby('PRICE').VOLUME.describe()
    data = [df[df.PRICE == 24990].VOLUME.describe().values.tolist(),
            df[df.PRICE == 25499].VOLUME.describe().values.tolist()]
    expected = pd.DataFrame(data,
                            index=pd.Index([24990, 25499], name='PRICE'),
                            columns=['count', 'mean', 'std', 'min',
                                     '25%', '50%', '75%', 'max'])
    tm.assert_frame_equal(result, expected)

# nunique
# --------------------------------

@pytest.mark.parametrize('n', 10 ** np.arange(2, 6))
@pytest.mark.parametrize('m', [10, 100, 1000])
@pytest.mark.parametrize('sort', [False, True])
@pytest.mark.parametrize('dropna', [False, True])
def test_series_groupby_nunique(n, m, sort, dropna):

    def check_nunique(df, keys, as_index=True):
        gr = df.groupby(keys, as_index=as_index, sort=sort)
        left = gr['julie'].nunique(dropna=dropna)

        gr = df.groupby(keys, as_index=as_index, sort=sort)
        right = gr['julie'].apply(Series.nunique, dropna=dropna)
        if not as_index:
            right = right.reset_index(drop=True)

        tm.assert_series_equal(left, right, check_names=False)

    days = date_range('2015-08-23', periods=10)

    frame = DataFrame({'jim': np.random.choice(list(ascii_lowercase), n),
                       'joe': np.random.choice(days, n),
                       'julie': np.random.randint(0, m, n)})

    check_nunique(frame, ['jim'])
    check_nunique(frame, ['jim', 'joe'])

    frame.loc[1::17, 'jim'] = None
    frame.loc[3::37, 'joe'] = None
    frame.loc[7::19, 'julie'] = None
    frame.loc[8::19, 'julie'] = None
    frame.loc[9::19, 'julie'] = None

    check_nunique(frame, ['jim'])
    check_nunique(frame, ['jim', 'joe'])
    check_nunique(frame, ['jim'], as_index=False)
    check_nunique(frame, ['jim', 'joe'], as_index=False)

def test_nunique():
    df = DataFrame({
        'A': list('abbacc'),
        'B': list('abxacc'),
        'C': list('abbacx'),
    })

    expected = DataFrame({'A': [1] * 3, 'B': [1, 2, 1], 'C': [1, 1, 2]})
    result = df.groupby('A', as_index=False).nunique()
    tm.assert_frame_equal(result, expected)

    # as_index
    expected.index = list('abc')
    expected.index.name = 'A'
    result = df.groupby('A').nunique()
    tm.assert_frame_equal(result, expected)

    # with na
    result = df.replace({'x': None}).groupby('A').nunique(dropna=False)
    tm.assert_frame_equal(result, expected)

    # dropna
    expected = DataFrame({'A': [1] * 3, 'B': [1] * 3, 'C': [1] * 3},
                         index=list('abc'))
    expected.index.name = 'A'
    result = df.replace({'x': None}).groupby('A').nunique()
    tm.assert_frame_equal(result, expected)

def test_nunique_with_object():
    # GH 11077
    data = pd.DataFrame(
        [[100, 1, 'Alice'],
         [200, 2, 'Bob'],
         [300, 3, 'Charlie'],
         [-400, 4, 'Dan'],
         [500, 5, 'Edith']],
        columns=['amount', 'id', 'name']
    )

    result = data.groupby(['id', 'amount'])['name'].nunique()
    index = MultiIndex.from_arrays([data.id, data.amount])
    expected = pd.Series([1] * 5, name='name', index=index)
    tm.assert_series_equal(result, expected)

def test_nunique_with_empty_series():
    # GH 12553
    data = pd.Series(name='name')
    result = data.groupby(level=0).nunique()
    expected = pd.Series(name='name', dtype='int64')
    tm.assert_series_equal(result, expected)

def test_nunique_with_timegrouper():
    # GH 13453
    test = pd.DataFrame({
        'time': [Timestamp('2016-06-28 09:35:35'),
                 Timestamp('2016-06-28 16:09:30'),
                 Timestamp('2016-06-28 16:46:28')],
        'data': ['1', '2', '3']}).set_index('time')
    result = test.groupby(pd.Grouper(freq='h'))['data'].nunique()
    expected = test.groupby(
        pd.Grouper(freq='h')
    )['data'].apply(pd.Series.nunique)
    tm.assert_series_equal(result, expected)

# count
# --------------------------------

def test_groupby_timedelta_cython_count():
    df = DataFrame({'g': list('ab' * 2),
                    'delt': np.arange(4).astype('timedelta64[ns]')})
    expected = Series([
        2, 2
    ], index=pd.Index(['a', 'b'], name='g'), name='delt')
    result = df.groupby('g').delt.count()
    tm.assert_series_equal(expected, result)

def test_count():
    n = 1 << 15
    dr = date_range('2015-08-30', periods=n // 10, freq='T')

    df = DataFrame({
        '1st': np.random.choice(
            list(ascii_lowercase), n),
        '2nd': np.random.randint(0, 5, n),
        '3rd': np.random.randn(n).round(3),
        '4th': np.random.randint(-10, 10, n),
        '5th': np.random.choice(dr, n),
        '6th': np.random.randn(n).round(3),
        '7th': np.random.randn(n).round(3),
        '8th': np.random.choice(dr, n) - np.random.choice(dr, 1),
        '9th': np.random.choice(
            list(ascii_lowercase), n)
    })

    for col in df.columns.drop(['1st', '2nd', '4th']):
        df.loc[np.random.choice(n, n // 10), col] = np.nan

    df['9th'] = df['9th'].astype('category')

    for key in '1st', '2nd', ['1st', '2nd']:
        left = df.groupby(key).count()
        right = df.groupby(key).apply(DataFrame.count).drop(key, axis=1)
        tm.assert_frame_equal(left, right)

    # GH5610
    # count counts non-nulls
    df = pd.DataFrame([[1, 2, 'foo'],
                       [1, np.nan, 'bar'],
                       [3, np.nan, np.nan]],
                      columns=['A', 'B', 'C'])

    count_as = df.groupby('A').count()
    count_not_as = df.groupby('A', as_index=False).count()

    expected = DataFrame([[1, 2], [0, 0]], columns=['B', 'C'],
                         index=[1, 3])
    expected.index.name = 'A'
    tm.assert_frame_equal(count_not_as, expected.reset_index())
    tm.assert_frame_equal(count_as, expected)

    count_B = df.groupby('A')['B'].count()
    tm.assert_series_equal(count_B, expected['B'])

def test_count_object():
    df = pd.DataFrame({'a': ['a'] * 3 + ['b'] * 3, 'c': [2] * 3 + [3] * 3})
    result = df.groupby('c').a.count()
    expected = pd.Series([
        3, 3
    ], index=pd.Index([2, 3], name='c'), name='a')
    tm.assert_series_equal(result, expected)

    df = pd.DataFrame({'a': ['a', np.nan, np.nan] + ['b'] * 3,
                       'c': [2] * 3 + [3] * 3})
    result = df.groupby('c').a.count()
    expected = pd.Series([
        1, 3
    ], index=pd.Index([2, 3], name='c'), name='a')
    tm.assert_series_equal(result, expected)

def test_count_cross_type():
    # GH8169
    vals = np.hstack((np.random.randint(0, 5, (100, 2)), np.random.randint(
        0, 2, (100, 2))))

    df = pd.DataFrame(vals, columns=['a', 'b', 'c', 'd'])
    df[df == 2] = np.nan
    expected = df.groupby(['c', 'd']).count()

    for t in ['float32', 'object']:
        df['a'] = df['a'].astype(t)
        df['b'] = df['b'].astype(t)
        result = df.groupby(['c', 'd']).count()
        tm.assert_frame_equal(result, expected)

def test_lower_int_prec_count():
    df = DataFrame({'a': np.array(
        [0, 1, 2, 100], np.int8),
        'b': np.array(
        [1, 2, 3, 6], np.uint32),
        'c': np.array(
        [4, 5, 6, 8], np.int16),
        'grp': list('ab' * 2)})
    result = df.groupby('grp').count()
    expected = DataFrame({'a': [2, 2],
                          'b': [2, 2],
                          'c': [2, 2]}, index=pd.Index(list('ab'),
                                                       name='grp'))
    tm.assert_frame_equal(result, expected)

def test_count_uses_size_on_exception():
    class RaisingObjectException(Exception):
        pass

    class RaisingObject(object):

        def __init__(self, msg='I will raise inside Cython'):
            super(RaisingObject, self).__init__()
            self.msg = msg

        def __eq__(self, other):
            # gets called in Cython to check that raising calls the method
            raise RaisingObjectException(self.msg)

    df = DataFrame({'a': [RaisingObject() for _ in range(4)],
                    'grp': list('ab' * 2)})
    result = df.groupby('grp').count()
    expected = DataFrame({'a': [2, 2]}, index=pd.Index(
        list('ab'), name='grp'))
    tm.assert_frame_equal(result, expected)

# size
# --------------------------------

def test_size(df):
    grouped = df.groupby(['A', 'B'])
    result = grouped.size()
    for key, group in grouped:
        assert result[key] == len(group)

    grouped = df.groupby('A')
    result = grouped.size()
    for key, group in grouped:
        assert result[key] == len(group)

    grouped = df.groupby('B')
    result = grouped.size()
    for key, group in grouped:
        assert result[key] == len(group)

    df = DataFrame(np.random.choice(20, (1000, 3)), columns=list('abc'))
    for sort, key in cart_product((False, True), ('a', 'b', ['a', 'b'])):
        left = df.groupby(key, sort=sort).size()
        right = df.groupby(key, sort=sort)['c'].apply(lambda a: a.shape[0])
        tm.assert_series_equal(left, right, check_names=False)

    # GH11699
    df = DataFrame([], columns=['A', 'B'])
    out = Series([], dtype='int64', index=Index([], name='A'))
    tm.assert_series_equal(df.groupby('A').size(), out)

# pipe
# --------------------------------

def test_pipe():
    # Test the pipe method of DataFrameGroupBy.
    # Issue #17871

    random_state = np.random.RandomState(1234567890)

    df = DataFrame({'A': ['foo', 'bar', 'foo', 'bar',
                          'foo', 'bar', 'foo', 'foo'],
                    'B': random_state.randn(8),
                    'C': random_state.randn(8)})

    def f(dfgb):
        return dfgb.B.max() - dfgb.C.min().min()

    def square(srs):
        return srs ** 2

    # Note that the transformations are
    # GroupBy -> Series
    # Series -> Series
    # This then chains the GroupBy.pipe and the
    # NDFrame.pipe methods
    result = df.groupby('A').pipe(f).pipe(square)

    index = Index([u'bar', u'foo'], dtype='object', name=u'A')
    expected = pd.Series([8.99110003361, 8.17516964785], name='B',
                         index=index)

    tm.assert_series_equal(expected, result)

def test_pipe_args():
    # Test passing args to the pipe method of DataFrameGroupBy.
    # Issue #17871

    df = pd.DataFrame({'group': ['A', 'A', 'B', 'B', 'C'],
                       'x': [1.0, 2.0, 3.0, 2.0, 5.0],
                       'y': [10.0, 100.0, 1000.0, -100.0, -1000.0]})

    def f(dfgb, arg1):
        return (dfgb.filter(lambda grp: grp.y.mean() > arg1, dropna=False)
                    .groupby(dfgb.grouper))

    def g(dfgb, arg2):
        return dfgb.sum() / dfgb.sum().sum() + arg2

    def h(df, arg3):
        return df.x + df.y - arg3

    result = (df
              .groupby('group')
              .pipe(f, 0)
              .pipe(g, 10)
              .pipe(h, 100))

    # Assert the results here
    index = pd.Index(['A', 'B', 'C'], name='group')
    expected = pd.Series([-79.5160891089, -78.4839108911, -80],
                         index=index)

    tm.assert_series_equal(expected, result)

    # test SeriesGroupby.pipe
    ser = pd.Series([1, 1, 2, 2, 3, 3])
    result = ser.groupby(ser).pipe(lambda grp: grp.sum() * grp.count())

    expected = pd.Series([4, 8, 12], index=pd.Int64Index([1, 2, 3]))

    tm.assert_series_equal(result, expected)

def test_groupby_mean_no_overflow():
    # Regression test for (#22487)
    df = pd.DataFrame({
        "user": ["A", "A", "A", "A", "A"],
        "connections": [4970, 4749, 4719, 4704, 18446744073699999744]
    })
    assert df.groupby('user')['connections'].mean()['A'] == 3689348814740003840


import numpy as np
from sklearn.utils.testing import (assert_allclose, assert_raises,
                                   assert_equal)
from sklearn.neighbors import KernelDensity, KDTree, NearestNeighbors
from sklearn.neighbors.ball_tree import kernel_norm
from sklearn.pipeline import make_pipeline
from sklearn.datasets import make_blobs
from sklearn.grid_search import GridSearchCV
from sklearn.preprocessing import StandardScaler

def compute_kernel_slow(Y, X, kernel, h):
    d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
    norm = kernel_norm(h, X.shape[1], kernel) / X.shape[0]

    if kernel == 'gaussian':
        return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
    elif kernel == 'tophat':
        return norm * (d < h).sum(-1)
    elif kernel == 'epanechnikov':
        return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
    elif kernel == 'exponential':
        return norm * (np.exp(-d / h)).sum(-1)
    elif kernel == 'linear':
        return norm * ((1 - d / h) * (d < h)).sum(-1)
    elif kernel == 'cosine':
        return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
    else:
        raise ValueError('kernel not recognized')

def test_kernel_density(n_samples=100, n_features=3):
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features)
    Y = rng.randn(n_samples, n_features)

    for kernel in ['gaussian', 'tophat', 'epanechnikov',
                   'exponential', 'linear', 'cosine']:
        for bandwidth in [0.01, 0.1, 1]:
            dens_true = compute_kernel_slow(Y, X, kernel, bandwidth)

            def check_results(kernel, bandwidth, atol, rtol):
                kde = KernelDensity(kernel=kernel, bandwidth=bandwidth,
                                    atol=atol, rtol=rtol)
                log_dens = kde.fit(X).score_samples(Y)
                assert_allclose(np.exp(log_dens), dens_true,
                                atol=atol, rtol=max(1E-7, rtol))
                assert_allclose(np.exp(kde.score(Y)),
                                np.prod(dens_true),
                                atol=atol, rtol=max(1E-7, rtol))

            for rtol in [0, 1E-5]:
                for atol in [1E-6, 1E-2]:
                    for breadth_first in (True, False):
                        yield (check_results, kernel, bandwidth, atol, rtol)

def test_kernel_density_sampling(n_samples=100, n_features=3):
    rng = np.random.RandomState(0)
    X = rng.randn(n_samples, n_features)

    bandwidth = 0.2

    for kernel in ['gaussian', 'tophat']:
        # draw a tophat sample
        kde = KernelDensity(bandwidth, kernel=kernel).fit(X)
        samp = kde.sample(100)
        assert_equal(X.shape, samp.shape)

        # check that samples are in the right range
        nbrs = NearestNeighbors(n_neighbors=1).fit(X)
        dist, ind = nbrs.kneighbors(X, return_distance=True)

        if kernel == 'tophat':
            assert np.all(dist < bandwidth)
        elif kernel == 'gaussian':
            # 5 standard deviations is safe for 100 samples, but there's a
            # very small chance this test could fail.
            assert np.all(dist < 5 * bandwidth)

    # check unsupported kernels
    for kernel in ['epanechnikov', 'exponential', 'linear', 'cosine']:
        kde = KernelDensity(bandwidth, kernel=kernel).fit(X)
        assert_raises(NotImplementedError, kde.sample, 100)

    # non-regression test: used to return a scalar
    X = rng.randn(4, 1)
    kde = KernelDensity(kernel="gaussian").fit(X)
    assert_equal(kde.sample().shape, (1, 1))

def test_kde_algorithm_metric_choice():
    # Smoke test for various metrics and algorithms
    rng = np.random.RandomState(0)
    X = rng.randn(10, 2)    # 2 features required for haversine dist.
    Y = rng.randn(10, 2)

    for algorithm in ['auto', 'ball_tree', 'kd_tree']:
        for metric in ['euclidean', 'minkowski', 'manhattan',
                       'chebyshev', 'haversine']:
            if algorithm == 'kd_tree' and metric not in KDTree.valid_metrics:
                assert_raises(ValueError, KernelDensity,
                              algorithm=algorithm, metric=metric)
            else:
                kde = KernelDensity(algorithm=algorithm, metric=metric)
                kde.fit(X)
                y_dens = kde.score_samples(Y)
                assert_equal(y_dens.shape, Y.shape[:1])

def test_kde_score(n_samples=100, n_features=3):
    pass
    #FIXME
    #np.random.seed(0)
    #X = np.random.random((n_samples, n_features))
    #Y = np.random.random((n_samples, n_features))

def test_kde_badargs():
    assert_raises(ValueError, KernelDensity,
                  algorithm='blah')
    assert_raises(ValueError, KernelDensity,
                  bandwidth=0)
    assert_raises(ValueError, KernelDensity,
                  kernel='blah')
    assert_raises(ValueError, KernelDensity,
                  metric='blah')
    assert_raises(ValueError, KernelDensity,
                  algorithm='kd_tree', metric='blah')

def test_kde_pipeline_gridsearch():
    # test that kde plays nice in pipelines and grid-searches
    X, _ = make_blobs(cluster_std=.1, random_state=1,
                      centers=[[0, 1], [1, 0], [0, 0]])
    pipe1 = make_pipeline(StandardScaler(with_mean=False, with_std=False),
                          KernelDensity(kernel="gaussian"))
    params = dict(kerneldensity__bandwidth=[0.001, 0.01, 0.1, 1, 10])
    search = GridSearchCV(pipe1, param_grid=params, cv=5)
    search.fit(X)
    assert_equal(search.best_params_['kerneldensity__bandwidth'], .1)


# -*- coding: utf-8 -*-
from datetime import datetime as dt

import numpy as np
import pandas as pd
import pytest
import six
from mock import patch, call, Mock
from numpy.testing.utils import assert_array_equal
from pandas import DatetimeIndex
from pandas.util.testing import assert_frame_equal
from pymongo import ReadPreference

from arctic._util import mongo_count
from arctic.date import DateRange, mktz, CLOSED_CLOSED, CLOSED_OPEN, OPEN_CLOSED, OPEN_OPEN
from arctic.exceptions import NoDataFoundException

def test_read(tickstore_lib):
    data = [{'ASK': 1545.25,
                  'ASKSIZE': 1002.0,
                  'BID': 1545.0,
                  'BIDSIZE': 55.0,
                  'CUMVOL': 2187387.0,
                  'DELETED_TIME': 0,
                  'INSTRTYPE': 'FUT',
                  'PRICE': 1545.0,
                  'SIZE': 1.0,
                  'TICK_STATUS': 0,
                  'TRADEHIGH': 1561.75,
                  'TRADELOW': 1537.25,
                  'index': 1185076787070},
                 {'CUMVOL': 354.0,
                  'DELETED_TIME': 0,
                  'PRICE': 1543.75,
                  'SIZE': 354.0,
                  'TRADEHIGH': 1543.75,
                  'TRADELOW': 1543.75,
                  'index': 1185141600600}]
    tickstore_lib.write('FEED::SYMBOL', data)

    df = tickstore_lib.read('FEED::SYMBOL', columns=['BID', 'ASK', 'PRICE'])

    assert_array_equal(df['ASK'].values, np.array([1545.25, np.nan]))
    assert_array_equal(df['BID'].values, np.array([1545, np.nan]))
    assert_array_equal(df['PRICE'].values, np.array([1545, 1543.75]))
    assert_array_equal(df.index.values.astype('object'), np.array([1185076787070000000, 1185141600600000000]))
    assert tickstore_lib._collection.find_one()['c'] == 2
    assert df.index.tzinfo == mktz()

def test_read_data_is_modifiable(tickstore_lib):
    data = [{'ASK': 1545.25,
                  'ASKSIZE': 1002.0,
                  'BID': 1545.0,
                  'BIDSIZE': 55.0,
                  'CUMVOL': 2187387.0,
                  'DELETED_TIME': 0,
                  'INSTRTYPE': 'FUT',
                  'PRICE': 1545.0,
                  'SIZE': 1.0,
                  'TICK_STATUS': 0,
                  'TRADEHIGH': 1561.75,
                  'TRADELOW': 1537.25,
                  'index': 1185076787070},
                 {'CUMVOL': 354.0,
                  'DELETED_TIME': 0,
                  'PRICE': 1543.75,
                  'SIZE': 354.0,
                  'TRADEHIGH': 1543.75,
                  'TRADELOW': 1543.75,
                  'index': 1185141600600}]
    tickstore_lib.write('FEED::SYMBOL', data)

    df = tickstore_lib.read('FEED::SYMBOL', columns=['BID', 'ASK', 'PRICE'])

    df[['BID', 'ASK', 'PRICE']] = 7
    assert np.all(df[['BID', 'ASK', 'PRICE']].values == np.array([[7, 7, 7], [7, 7, 7]]))

def test_read_allow_secondary(tickstore_lib):
    data = [{'ASK': 1545.25,
                  'ASKSIZE': 1002.0,
                  'BID': 1545.0,
                  'BIDSIZE': 55.0,
                  'CUMVOL': 2187387.0,
                  'DELETED_TIME': 0,
                  'INSTRTYPE': 'FUT',
                  'PRICE': 1545.0,
                  'SIZE': 1.0,
                  'TICK_STATUS': 0,
                  'TRADEHIGH': 1561.75,
                  'TRADELOW': 1537.25,
                  'index': 1185076787070},
                 {'CUMVOL': 354.0,
                  'DELETED_TIME': 0,
                  'PRICE': 1543.75,
                  'SIZE': 354.0,
                  'TRADEHIGH': 1543.75,
                  'TRADELOW': 1543.75,
                  'index': 1185141600600}]
    tickstore_lib.write('FEED::SYMBOL', data)

    with patch('pymongo.collection.Collection.find', side_effect=tickstore_lib._collection.find) as find:
        with patch('pymongo.collection.Collection.with_options', side_effect=tickstore_lib._collection.with_options) as with_options:
            with patch.object(tickstore_lib, '_read_preference', side_effect=tickstore_lib._read_preference) as read_pref:
                df = tickstore_lib.read('FEED::SYMBOL', columns=['BID', 'ASK', 'PRICE'], allow_secondary=True)
    assert read_pref.call_args_list == [call(True)]
    assert with_options.call_args_list == [call(read_preference=ReadPreference.NEAREST)]
    assert find.call_args_list == [call({'sy': 'FEED::SYMBOL'}, sort=[('s', 1)], projection={'s': 1, '_id': 0}),
                                   call({'sy': 'FEED::SYMBOL', 's': {'$lte': dt(2007, 8, 21, 3, 59, 47, 70000)}},
                                        projection={'sy': 1, 'cs.PRICE': 1, 'i': 1, 'cs.BID': 1, 's': 1, 'im': 1, 'v': 1, 'cs.ASK': 1})]

    assert_array_equal(df['ASK'].values, np.array([1545.25, np.nan]))
    assert tickstore_lib._collection.find_one()['c'] == 2

def test_read_symbol_as_column(tickstore_lib):
    data = [{'ASK': 1545.25,
             'index': 1185076787070},
            {'CUMVOL': 354.0,
             'index': 1185141600600}]
    tickstore_lib.write('FEED::SYMBOL', data)

    df = tickstore_lib.read('FEED::SYMBOL', columns=['SYMBOL', 'CUMVOL'])
    assert all(df['SYMBOL'].values == ['FEED::SYMBOL'])

def test_read_multiple_symbols(tickstore_lib):
    data1 = [{'ASK': 1545.25,
                  'ASKSIZE': 1002.0,
                  'BID': 1545.0,
                  'BIDSIZE': 55.0,
                  'CUMVOL': 2187387.0,
                  'DELETED_TIME': 0,
                  'INSTRTYPE': 'FUT',
                  'PRICE': 1545.0,
                  'SIZE': 1.0,
                  'TICK_STATUS': 0,
                  'TRADEHIGH': 1561.75,
                  'TRADELOW': 1537.25,
                  'index': 1185076787070}, ]
    data2 = [{'CUMVOL': 354.0,
                  'DELETED_TIME': 0,
                  'PRICE': 1543.75,
                  'SIZE': 354.0,
                  'TRADEHIGH': 1543.75,
                  'TRADELOW': 1543.75,
                  'index': 1185141600600}]

    tickstore_lib.write('BAR', data2)
    tickstore_lib.write('FOO', data1)

    df = tickstore_lib.read(['FOO', 'BAR'], columns=['BID', 'ASK', 'PRICE'])

    assert all(df['SYMBOL'].values == ['FOO', 'BAR'])
    assert_array_equal(df['ASK'].values, np.array([1545.25, np.nan]))
    assert_array_equal(df['BID'].values, np.array([1545, np.nan]))
    assert_array_equal(df['PRICE'].values, np.array([1545, 1543.75]))
    assert_array_equal(df.index.values.astype('object'), np.array([1185076787070000000, 1185141600600000000]))
    assert tickstore_lib._collection.find_one()['c'] == 1

@pytest.mark.parametrize('chunk_size', [1, 100])
def test_read_all_cols_all_dtypes(tickstore_lib, chunk_size):
    data = [{'f': 0.1,
            'of': 0.2,
            's': 's',
            'os': 'os',
            'l': 1,
            'ol': 2,
            'index': dt(1970, 1, 1, tzinfo=mktz('UTC')),
            },
            {'f': 0.3,
            'nf': 0.4,
            's': 't',
            'ns': 'ns',
            'l': 3,
            'nl': 4,
            'index': dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC')),
            },
            ]
    tickstore_lib._chunk_size = chunk_size
    tickstore_lib.write('sym', data)
    df = tickstore_lib.read('sym', columns=None)

    assert df.index.tzinfo == mktz()

    # The below is probably more trouble than it's worth, but we *should*
    # be able to roundtrip data and get the same answer...

    # Ints become floats
    data[0]['l'] = float(data[0]['l'])
    # Treat missing strings as None
    data[0]['ns'] = None
    data[1]['os'] = None
    index = DatetimeIndex([dt(1970, 1, 1, tzinfo=mktz('UTC')),
                         dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC'))],
                        )
    df.index = df.index.tz_convert(mktz('UTC'))
    expected = pd.DataFrame(data, index=index)
    expected = expected[df.columns]
    assert_frame_equal(expected, df, check_names=False)

DUMMY_DATA = [
              {'a': 1.,
               'b': 2.,
               'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London'))
               },
              {'b': 3.,
               'c': 4.,
               'index': dt(2013, 1, 2, tzinfo=mktz('Europe/London'))
               },
              {'b': 5.,
               'c': 6.,
               'index': dt(2013, 1, 3, tzinfo=mktz('Europe/London'))
               },
              {'b': 7.,
               'c': 8.,
               'index': dt(2013, 1, 4, tzinfo=mktz('Europe/London'))
               },
              {'b': 9.,
               'c': 10.,
               'index': dt(2013, 1, 5, tzinfo=mktz('Europe/London'))
               },
              ]

def test_date_range(tickstore_lib):
    tickstore_lib.write('SYM', DUMMY_DATA)
    df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130103), columns=None)
    assert_array_equal(df['a'].values, np.array([1, np.nan, np.nan]))
    assert_array_equal(df['b'].values, np.array([2., 3., 5.]))
    assert_array_equal(df['c'].values, np.array([np.nan, 4., 6.]))

    tickstore_lib.delete('SYM')

    # Chunk every 3 symbols and lets have some fun
    tickstore_lib._chunk_size = 3
    tickstore_lib.write('SYM', DUMMY_DATA)

    with patch('pymongo.collection.Collection.find', side_effect=tickstore_lib._collection.find) as f:
        df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130103), columns=None)
        assert_array_equal(df['b'].values, np.array([2., 3., 5.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 1
        df = tickstore_lib.read('SYM', date_range=DateRange(20130102, 20130103), columns=None)
        assert_array_equal(df['b'].values, np.array([3., 5.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 1
        df = tickstore_lib.read('SYM', date_range=DateRange(20130103, 20130103), columns=None)
        assert_array_equal(df['b'].values, np.array([5.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 1

        df = tickstore_lib.read('SYM', date_range=DateRange(20130102, 20130104), columns=None)
        assert_array_equal(df['b'].values, np.array([3., 5., 7.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 2
        df = tickstore_lib.read('SYM', date_range=DateRange(20130102, 20130105), columns=None)
        assert_array_equal(df['b'].values, np.array([3., 5., 7., 9.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 2

        df = tickstore_lib.read('SYM', date_range=DateRange(20130103, 20130104), columns=None)
        assert_array_equal(df['b'].values, np.array([5., 7.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 2
        df = tickstore_lib.read('SYM', date_range=DateRange(20130103, 20130105), columns=None)
        assert_array_equal(df['b'].values, np.array([5., 7., 9.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 2

        df = tickstore_lib.read('SYM', date_range=DateRange(20130104, 20130105), columns=None)
        assert_array_equal(df['b'].values, np.array([7., 9.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 1

        # Test the different open-closed behaviours
        df = tickstore_lib.read('SYM', date_range=DateRange(20130104, 20130105, CLOSED_CLOSED), columns=None)
        assert_array_equal(df['b'].values, np.array([7., 9.]))
        df = tickstore_lib.read('SYM', date_range=DateRange(20130104, 20130105, CLOSED_OPEN), columns=None)
        assert_array_equal(df['b'].values, np.array([7.]))
        df = tickstore_lib.read('SYM', date_range=DateRange(20130104, 20130105, OPEN_CLOSED), columns=None)
        assert_array_equal(df['b'].values, np.array([9.]))
        df = tickstore_lib.read('SYM', date_range=DateRange(20130104, 20130105, OPEN_OPEN), columns=None)
        assert_array_equal(df['b'].values, np.array([]))

def test_date_range_end_not_in_range(tickstore_lib):
    DUMMY_DATA = [
                  {'a': 1.,
                   'b': 2.,
                   'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London'))
                   },
                  {'b': 3.,
                   'c': 4.,
                   'index': dt(2013, 1, 2, 10, 1, tzinfo=mktz('Europe/London'))
                   },
                  ]

    tickstore_lib._chunk_size = 1
    tickstore_lib.write('SYM', DUMMY_DATA)
    with patch.object(tickstore_lib._collection, 'find', side_effect=tickstore_lib._collection.find) as f:
        df = tickstore_lib.read('SYM', date_range=DateRange(20130101, dt(2013, 1, 2, 9, 0)), columns=None)
        assert_array_equal(df['b'].values, np.array([2.]))
        assert mongo_count(tickstore_lib._collection, filter=f.call_args_list[-1][0][0]) == 1

@pytest.mark.parametrize('tz_name', ['UTC',
                                     'Europe/London',  # Sometimes ahead of UTC
                                     'America/New_York',  # Behind UTC
                                      ])
def test_date_range_default_timezone(tickstore_lib, tz_name):
    """
    We assume naive datetimes are user-local
    """
    DUMMY_DATA = [
                  {'a': 1.,
                   'b': 2.,
                   'index': dt(2013, 1, 1, tzinfo=mktz(tz_name))
                   },
                  # Half-way through the year
                  {'b': 3.,
                   'c': 4.,
                   'index': dt(2013, 7, 1, tzinfo=mktz(tz_name))
                   },
                  ]

    with patch('tzlocal.get_localzone', return_value=Mock(zone=tz_name)):
        tickstore_lib._chunk_size = 1
        tickstore_lib.write('SYM', DUMMY_DATA)
        df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130701), columns=None)

        assert df.index.tzinfo == mktz()

        assert len(df) == 2
        assert df.index[1] == dt(2013, 7, 1, tzinfo=mktz(tz_name))
        df = tickstore_lib.read('SYM', date_range=DateRange(20130101, 20130101), columns=None)
        assert len(df) == 1
        assert df.index.tzinfo == mktz()

        df = tickstore_lib.read('SYM', date_range=DateRange(20130701, 20130701), columns=None)
        assert len(df) == 1
        assert df.index.tzinfo == mktz()

def test_date_range_no_bounds(tickstore_lib):
    DUMMY_DATA = [
                  {'a': 1.,
                   'b': 2.,
                   'index': dt(2013, 1, 1, tzinfo=mktz('Europe/London'))
                   },
                  {'a': 3.,
                   'b': 4.,
                   'index': dt(2013, 1, 30, tzinfo=mktz('Europe/London'))
                   },
                  {'b': 5.,
                   'c': 6.,
                   'index': dt(2013, 2, 2, 10, 1, tzinfo=mktz('Europe/London'))
                   },
                  ]

    tickstore_lib._chunk_size = 1
    tickstore_lib.write('SYM', DUMMY_DATA)

    # 1) No start, no end
    df = tickstore_lib.read('SYM', columns=None)
    assert_array_equal(df['b'].values, np.array([2., 4.]))
    # 1.2) Start before the real start
    df = tickstore_lib.read('SYM', date_range=DateRange(20121231), columns=None)
    assert_array_equal(df['b'].values, np.array([2., 4.]))
    # 2.1) Only go one month out
    df = tickstore_lib.read('SYM', date_range=DateRange(20130101), columns=None)
    assert_array_equal(df['b'].values, np.array([2., 4.]))
    # 2.2) Only go one month out
    df = tickstore_lib.read('SYM', date_range=DateRange(20130102), columns=None)
    assert_array_equal(df['b'].values, np.array([4.]))
    # 3) No start
    df = tickstore_lib.read('SYM', date_range=DateRange(end=20130102), columns=None)
    assert_array_equal(df['b'].values, np.array([2.]))
    # 4) Outside bounds
    df = tickstore_lib.read('SYM', date_range=DateRange(end=20131212), columns=None)
    assert_array_equal(df['b'].values, np.array([2., 4., 5.]))

def test_date_range_BST(tickstore_lib):
    DUMMY_DATA = [
                  {'a': 1.,
                   'b': 2.,
                   'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('Europe/London'))
                   },
                  {'a': 3.,
                   'b': 4.,
                   'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London'))
                   },
                  ]
    tickstore_lib._chunk_size = 1
    tickstore_lib.write('SYM', DUMMY_DATA)

    df = tickstore_lib.read('SYM', columns=None)
    assert_array_equal(df['b'].values, np.array([2., 4.]))

#     df = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, 12),
#                                                                       dt(2013, 6, 1, 13)))
#     assert_array_equal(df['b'].values, np.array([2., 4.]))
    df = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, 12, tzinfo=mktz('Europe/London')),
                                                                            dt(2013, 6, 1, 13, tzinfo=mktz('Europe/London'))))
    assert_array_equal(df['b'].values, np.array([2., 4.]))

    df = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, 12, tzinfo=mktz('UTC')),
                                                                            dt(2013, 6, 1, 13, tzinfo=mktz('UTC'))))
    assert_array_equal(df['b'].values, np.array([4., ]))

def test_read_no_data(tickstore_lib):
    with pytest.raises(NoDataFoundException):
        tickstore_lib.read('missing_sym', DateRange(20131212, 20131212))

def test_write_no_tz(tickstore_lib):
    DUMMY_DATA = [
                  {'a': 1.,
                   'b': 2.,
                   'index': dt(2013, 6, 1, 12, 00)
                   }]
    with pytest.raises(ValueError):
        tickstore_lib.write('SYM', DUMMY_DATA)

def test_read_out_of_order(tickstore_lib):
    data = [{'A': 120, 'D': 1}, {'A': 122, 'B': 2.0}, {'A': 3, 'B': 3.0, 'D': 1}]
    tick_index = [dt(2013, 6, 1, 12, 00, tzinfo=mktz('UTC')),
                  dt(2013, 6, 1, 11, 00, tzinfo=mktz('UTC')),  # Out-of-order
                  dt(2013, 6, 1, 13, 00, tzinfo=mktz('UTC'))]
    data = pd.DataFrame(data, index=tick_index)

    tickstore_lib._chunk_size = 3
    tickstore_lib.write('SYM', data)
    tickstore_lib.read('SYM', columns=None)
    assert len(tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 2, tzinfo=mktz('UTC'))))) == 3
    assert len(tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1, tzinfo=mktz('UTC')), dt(2013, 6, 1, 12, tzinfo=mktz('UTC'))))) == 2

def test_read_chunk_boundaries(tickstore_lib):
    SYM1_DATA = [
                  {'a': 1.,
                   'b': 2.,
                   'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('UTC'))
                   },
                   {'a': 3.,
                   'b': 4.,
                   'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('UTC'))
                   },
                 # Chunk boundary here
                   {'a': 5.,
                   'b': 6.,
                   'index': dt(2013, 6, 1, 14, 00, tzinfo=mktz('UTC'))
                   }
                  ]
    SYM2_DATA = [
                  {'a': 7.,
                   'b': 8.,
                   'index': dt(2013, 6, 1, 12, 30, tzinfo=mktz('UTC'))
                   },
                   {'a': 9.,
                   'b': 10.,
                   'index': dt(2013, 6, 1, 13, 30, tzinfo=mktz('UTC'))
                   },
                 # Chunk boundary here
                   {'a': 11.,
                   'b': 12.,
                   'index': dt(2013, 6, 1, 14, 30, tzinfo=mktz('UTC'))
                   }
                  ]
    tickstore_lib._chunk_size = 2
    tickstore_lib.write('SYM1', SYM1_DATA)
    tickstore_lib.write('SYM2', SYM2_DATA)

    assert len(tickstore_lib.read('SYM1', columns=None, date_range=DateRange(dt(2013, 6, 1, 12, 45, tzinfo=mktz('UTC')), dt(2013, 6, 1, 15, 00, tzinfo=mktz('UTC'))))) == 2
    assert len(tickstore_lib.read('SYM2', columns=None, date_range=DateRange(dt(2013, 6, 1, 12, 45, tzinfo=mktz('UTC')), dt(2013, 6, 1, 15, 00, tzinfo=mktz('UTC'))))) == 2

    assert len(tickstore_lib.read(['SYM1', 'SYM2'], columns=None, date_range=DateRange(dt(2013, 6, 1, 12, 45, tzinfo=mktz('UTC')), dt(2013, 6, 1, 15, 00, tzinfo=mktz('UTC'))))) == 4

def test_read_spanning_chunks(tickstore_lib):
    SYM1_DATA = [
              {'a': 1.,
               'b': 2.,
               'index': dt(2013, 6, 1, 11, 00, tzinfo=mktz('UTC'))
               },
               {'a': 3.,
               'b': 4.,
               'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('UTC'))
               },
             # Chunk boundary here
               {'a': 5.,
               'b': 6.,
               'index': dt(2013, 6, 1, 14, 00, tzinfo=mktz('UTC'))
               }
              ]
    SYM2_DATA = [
                  {'a': 7.,
                   'b': 8.,
                   'index': dt(2013, 6, 1, 12, 30, tzinfo=mktz('UTC'))
                   },
                   {'a': 9.,
                   'b': 10.,
                   'index': dt(2013, 6, 1, 13, 30, tzinfo=mktz('UTC'))
                   },
                 # Chunk boundary here
                   {'a': 11.,
                   'b': 12.,
                   'index': dt(2013, 6, 1, 14, 30, tzinfo=mktz('UTC'))
                   }
                  ]
    tickstore_lib._chunk_size = 2
    tickstore_lib.write('SYM1', SYM1_DATA)
    tickstore_lib.write('SYM2', SYM2_DATA)

    # Even though the latest chunk that's the closest to the start point for SYM1 starts at 11:00, it ends before the start point,
    # so we want to ignore it and start from SYM2 (12:30) instead.
    assert tickstore_lib._mongo_date_range_query(
                                                 ['SYM1', 'SYM2'],
                                                 date_range=DateRange(dt(2013, 6, 1, 12, 45, tzinfo=mktz('UTC')),
                                                                      dt(2013, 6, 1, 15, 00, tzinfo=mktz('UTC')))) == \
        {'s': {'$gte': dt(2013, 6, 1, 12, 30, tzinfo=mktz('UTC')), '$lte': dt(2013, 6, 1, 15, 0, tzinfo=mktz('UTC'))}}

def test_read_inside_range(tickstore_lib):
    SYM1_DATA = [
              {'a': 1.,
               'b': 2.,
               'index': dt(2013, 6, 1, 0, 00, tzinfo=mktz('UTC'))
               },
               {'a': 3.,
               'b': 4.,
               'index': dt(2013, 6, 1, 1, 00, tzinfo=mktz('UTC'))
               },
             # Chunk boundary here
               {'a': 5.,
               'b': 6.,
               'index': dt(2013, 6, 1, 14, 00, tzinfo=mktz('UTC'))
               }
              ]
    SYM2_DATA = [
                  {'a': 7.,
                   'b': 8.,
                   'index': dt(2013, 6, 1, 12, 30, tzinfo=mktz('UTC'))
                   },
                   {'a': 9.,
                   'b': 10.,
                   'index': dt(2013, 6, 1, 13, 30, tzinfo=mktz('UTC'))
                   },
                 # Chunk boundary here
                   {'a': 11.,
                   'b': 12.,
                   'index': dt(2013, 6, 1, 14, 30, tzinfo=mktz('UTC'))
                   }
                  ]
    tickstore_lib._chunk_size = 2
    tickstore_lib.write('SYM1', SYM1_DATA)
    tickstore_lib.write('SYM2', SYM2_DATA)

    # If there are no chunks spanning the range, we still cap the start range so that we don't
    # fetch SYM1's 0am--1am chunk
    assert tickstore_lib._mongo_date_range_query(
                                                 ['SYM1', 'SYM2'],
                                                 date_range=DateRange(dt(2013, 6, 1, 10, 0, tzinfo=mktz('UTC')),
                                                                      dt(2013, 6, 1, 15, 0, tzinfo=mktz('UTC')))) == \
        {'s': {'$gte': dt(2013, 6, 1, 10, 0, tzinfo=mktz('UTC')), '$lte': dt(2013, 6, 1, 15, 0, tzinfo=mktz('UTC'))}}

def test_read_longs(tickstore_lib):
    DUMMY_DATA = [
                  {'a': 1,
                   'index': dt(2013, 6, 1, 12, 00, tzinfo=mktz('Europe/London'))
                   },
                  {
                   'b': 4,
                   'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London'))
                   },
                  ]
    tickstore_lib._chunk_size = 3
    tickstore_lib.write('SYM', DUMMY_DATA)
    tickstore_lib.read('SYM', columns=None)
    read = tickstore_lib.read('SYM', columns=None, date_range=DateRange(dt(2013, 6, 1), dt(2013, 6, 2)))
    assert read['a'][0] == 1
    assert np.isnan(read['b'][0])

def test_read_with_image(tickstore_lib):
    DUMMY_DATA = [
              {'a': 1.,
               'index': dt(2013, 1, 1, 11, 00, tzinfo=mktz('Europe/London'))
               },
              {
               'b': 4.,
               'index': dt(2013, 1, 1, 12, 00, tzinfo=mktz('Europe/London'))
               },
              ]
    # Add an image
    tickstore_lib.write('SYM', DUMMY_DATA)
    tickstore_lib._collection.update_one({},
                                     {'$set':
                                      {'im': {'i':
                                              {'a': 37.,
                                               'c': 2.,
                                               },
                                              't': dt(2013, 1, 1, 10, tzinfo=mktz('Europe/London'))
                                              }
                                       }
                                      }
                                     )

    dr = DateRange(dt(2013, 1, 1), dt(2013, 1, 2))
    # tickstore_lib.read('SYM', columns=None)
    df = tickstore_lib.read('SYM', columns=None, date_range=dr)
    assert df['a'][0] == 1

    # Read with the image as well - all columns
    df = tickstore_lib.read('SYM', columns=None, date_range=dr, include_images=True)
    assert set(df.columns) == set(('a', 'b', 'c'))
    assert_array_equal(df['a'].values, np.array([37, 1, np.nan]))
    assert_array_equal(df['b'].values, np.array([np.nan, np.nan, 4]))
    assert_array_equal(df['c'].values, np.array([2, np.nan, np.nan]))
    assert df.index[0] == dt(2013, 1, 1, 10, tzinfo=mktz('Europe/London'))
    assert df.index[1] == dt(2013, 1, 1, 11, tzinfo=mktz('Europe/London'))
    assert df.index[2] == dt(2013, 1, 1, 12, tzinfo=mktz('Europe/London'))

    # Read just columns from the updates
    df = tickstore_lib.read('SYM', columns=('a', 'b'), date_range=dr, include_images=True)
    assert set(df.columns) == set(('a', 'b'))
    assert_array_equal(df['a'].values, np.array([37, 1, np.nan]))
    assert_array_equal(df['b'].values, np.array([np.nan, np.nan, 4]))
    assert df.index[0] == dt(2013, 1, 1, 10, tzinfo=mktz('Europe/London'))
    assert df.index[1] == dt(2013, 1, 1, 11, tzinfo=mktz('Europe/London'))
    assert df.index[2] == dt(2013, 1, 1, 12, tzinfo=mktz('Europe/London'))

    # Read one column from the updates
    df = tickstore_lib.read('SYM', columns=('a',), date_range=dr, include_images=True)
    assert set(df.columns) == set(('a',))
    assert_array_equal(df['a'].values, np.array([37, 1]))
    assert df.index[0] == dt(2013, 1, 1, 10, tzinfo=mktz('Europe/London'))
    assert df.index[1] == dt(2013, 1, 1, 11, tzinfo=mktz('Europe/London'))

    # Read just the image column
    df = tickstore_lib.read('SYM', columns=['c'], date_range=dr, include_images=True)
    assert set(df.columns) == set(['c'])
    assert_array_equal(df['c'].values, np.array([2]))
    assert df.index[0] == dt(2013, 1, 1, 10, tzinfo=mktz('Europe/London'))

def test_read_with_metadata(tickstore_lib):
    metadata = {'metadata': 'important data'}
    tickstore_lib.write('test', [{'index': dt(2013, 6, 1, 13, 00, tzinfo=mktz('Europe/London')), 'price': 100.50, 'ticker': 'QQQ'}], metadata=metadata)
    m = tickstore_lib.read_metadata('test')
    assert(metadata == m)

def test_read_strings(tickstore_lib):
    df = pd.DataFrame(data={'data': ['A', 'B', 'C']},
                      index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')),
                                           dt(2016, 1, 2, 00, tzinfo=mktz('UTC')),
                                           dt(2016, 1, 3, 00, tzinfo=mktz('UTC'))], name='date'))
    tickstore_lib.write('test', df)
    read_df = tickstore_lib.read('test')
    assert(all(read_df['data'].values == df['data'].values))

def test_read_utf8_strings(tickstore_lib):
    data = ['一', '二', '三'] # Chinese character [one, two , three]
    if six.PY2:
      utf8_data = data
      unicode_data = [s.decode('utf8') for s in data]
    else:
      utf8_data = [s.encode('utf8') for s in data]
      unicode_data = data
    df = pd.DataFrame(data={'data': utf8_data},
                      index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')),
                                           dt(2016, 1, 2, 00, tzinfo=mktz('UTC')),
                                           dt(2016, 1, 3, 00, tzinfo=mktz('UTC'))], name='date'))
    tickstore_lib.write('test', df)
    read_df = tickstore_lib.read('test')
    assert(all(read_df['data'].values == np.array(unicode_data)))

def test_read_unicode_strings(tickstore_lib):
    df = pd.DataFrame(data={'data': [u'一', u'二', u'三']}, # Chinese character [one, two , three]
                      index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')),
                                           dt(2016, 1, 2, 00, tzinfo=mktz('UTC')),
                                           dt(2016, 1, 3, 00, tzinfo=mktz('UTC'))], name='date'))
    tickstore_lib.write('test', df)
    read_df = tickstore_lib.read('test')
    assert(all(read_df['data'].values == df['data'].values))

def test_objects_fail(tickstore_lib):
    class Fake(object):
        def __init__(self, val):
            self.val = val

        def fake(self):
            return self.val

    df = pd.DataFrame(data={'data': [Fake(1), Fake(2)]},
                      index=pd.Index(data=[dt(2016, 1, 1, 00, tzinfo=mktz('UTC')),
                                           dt(2016, 1, 2, 00, tzinfo=mktz('UTC'))], name='date'))

    with pytest.raises(Exception) as e:
        tickstore_lib.write('test', df)
    assert('Casting object column to string failed' in str(e.value))


"""
Various bayesian regression
"""
from __future__ import print_function

# Authors: V. Michel, F. Pedregosa, A. Gramfort
# License: BSD 3 clause

from math import log
import numpy as np
from scipy import linalg

from .base import LinearModel
from ..base import RegressorMixin
from ..utils.extmath import fast_logdet, pinvh
from ..utils import check_X_y

###############################################################################
# BayesianRidge regression

class BayesianRidge(LinearModel, RegressorMixin):
    """Bayesian ridge regression

    Fit a Bayesian ridge model and optimize the regularization parameters
    lambda (precision of the weights) and alpha (precision of the noise).

    Read more in the :ref:`User Guide <bayesian_regression>`.

    Parameters
    ----------
    n_iter : int, optional
        Maximum number of iterations.  Default is 300.

    tol : float, optional
        Stop the algorithm if w has converged. Default is 1.e-3.

    alpha_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the alpha parameter. Default is 1.e-6

    alpha_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the alpha parameter.
        Default is 1.e-6.

    lambda_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the lambda parameter. Default is 1.e-6.

    lambda_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the lambda parameter.
        Default is 1.e-6

    compute_score : boolean, optional
        If True, compute the objective function at each step of the model.
        Default is False

    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).
        Default is True.

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.

    copy_X : boolean, optional, default True
        If True, X will be copied; else, it may be overwritten.

    verbose : boolean, optional, default False
        Verbose mode when fitting the model.

    Attributes
    ----------
    coef_ : array, shape = (n_features)
        Coefficients of the regression model (mean of distribution)

    alpha_ : float
       estimated precision of the noise.

    lambda_ : array, shape = (n_features)
       estimated precisions of the weights.

    scores_ : float
        if computed, value of the objective function (to be maximized)

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.BayesianRidge()
    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
    ... # doctest: +NORMALIZE_WHITESPACE
    BayesianRidge(alpha_1=1e-06, alpha_2=1e-06, compute_score=False,
            copy_X=True, fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06,
            n_iter=300, normalize=False, tol=0.001, verbose=False)
    >>> clf.predict([[1, 1]])
    array([ 1.])

    Notes
    -----
    See examples/linear_model/plot_bayesian_ridge.py for an example.
    """

    def __init__(self, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
                 lambda_1=1.e-6, lambda_2=1.e-6, compute_score=False,
                 fit_intercept=True, normalize=False, copy_X=True,
                 verbose=False):
        self.n_iter = n_iter
        self.tol = tol
        self.alpha_1 = alpha_1
        self.alpha_2 = alpha_2
        self.lambda_1 = lambda_1
        self.lambda_2 = lambda_2
        self.compute_score = compute_score
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.copy_X = copy_X
        self.verbose = verbose

    def fit(self, X, y):
        """Fit the model

        Parameters
        ----------
        X : numpy array of shape [n_samples,n_features]
            Training data
        y : numpy array of shape [n_samples]
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
        X, y, X_mean, y_mean, X_std = self._center_data(
            X, y, self.fit_intercept, self.normalize, self.copy_X)
        n_samples, n_features = X.shape

        ### Initialization of the values of the parameters
        alpha_ = 1. / np.var(y)
        lambda_ = 1.

        verbose = self.verbose
        lambda_1 = self.lambda_1
        lambda_2 = self.lambda_2
        alpha_1 = self.alpha_1
        alpha_2 = self.alpha_2

        self.scores_ = list()
        coef_old_ = None

        XT_y = np.dot(X.T, y)
        U, S, Vh = linalg.svd(X, full_matrices=False)
        eigen_vals_ = S ** 2

        ### Convergence loop of the bayesian ridge regression
        for iter_ in range(self.n_iter):

            ### Compute mu and sigma
            # sigma_ = lambda_ / alpha_ * np.eye(n_features) + np.dot(X.T, X)
            # coef_ = sigma_^-1 * XT * y
            if n_samples > n_features:
                coef_ = np.dot(Vh.T,
                               Vh / (eigen_vals_ + lambda_ / alpha_)[:, None])
                coef_ = np.dot(coef_, XT_y)
                if self.compute_score:
                    logdet_sigma_ = - np.sum(
                        np.log(lambda_ + alpha_ * eigen_vals_))
            else:
                coef_ = np.dot(X.T, np.dot(
                    U / (eigen_vals_ + lambda_ / alpha_)[None, :], U.T))
                coef_ = np.dot(coef_, y)
                if self.compute_score:
                    logdet_sigma_ = lambda_ * np.ones(n_features)
                    logdet_sigma_[:n_samples] += alpha_ * eigen_vals_
                    logdet_sigma_ = - np.sum(np.log(logdet_sigma_))

            ### Update alpha and lambda
            rmse_ = np.sum((y - np.dot(X, coef_)) ** 2)
            gamma_ = (np.sum((alpha_ * eigen_vals_)
                      / (lambda_ + alpha_ * eigen_vals_)))
            lambda_ = ((gamma_ + 2 * lambda_1)
                       / (np.sum(coef_ ** 2) + 2 * lambda_2))
            alpha_ = ((n_samples - gamma_ + 2 * alpha_1)
                      / (rmse_ + 2 * alpha_2))

            ### Compute the objective function
            if self.compute_score:
                s = lambda_1 * log(lambda_) - lambda_2 * lambda_
                s += alpha_1 * log(alpha_) - alpha_2 * alpha_
                s += 0.5 * (n_features * log(lambda_)
                            + n_samples * log(alpha_)
                            - alpha_ * rmse_
                            - (lambda_ * np.sum(coef_ ** 2))
                            - logdet_sigma_
                            - n_samples * log(2 * np.pi))
                self.scores_.append(s)

            ### Check for convergence
            if iter_ != 0 and np.sum(np.abs(coef_old_ - coef_)) < self.tol:
                if verbose:
                    print("Convergence after ", str(iter_), " iterations")
                break
            coef_old_ = np.copy(coef_)

        self.alpha_ = alpha_
        self.lambda_ = lambda_
        self.coef_ = coef_

        self._set_intercept(X_mean, y_mean, X_std)
        return self

###############################################################################
# ARD (Automatic Relevance Determination) regression

class ARDRegression(LinearModel, RegressorMixin):
    """Bayesian ARD regression.

    Fit the weights of a regression model, using an ARD prior. The weights of
    the regression model are assumed to be in Gaussian distributions.
    Also estimate the parameters lambda (precisions of the distributions of the
    weights) and alpha (precision of the distribution of the noise).
    The estimation is done by an iterative procedures (Evidence Maximization)

    Read more in the :ref:`User Guide <bayesian_regression>`.

    Parameters
    ----------
    n_iter : int, optional
        Maximum number of iterations. Default is 300

    tol : float, optional
        Stop the algorithm if w has converged. Default is 1.e-3.

    alpha_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the alpha parameter. Default is 1.e-6.

    alpha_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the alpha parameter. Default is 1.e-6.

    lambda_1 : float, optional
        Hyper-parameter : shape parameter for the Gamma distribution prior
        over the lambda parameter. Default is 1.e-6.

    lambda_2 : float, optional
        Hyper-parameter : inverse scale parameter (rate parameter) for the
        Gamma distribution prior over the lambda parameter. Default is 1.e-6.

    compute_score : boolean, optional
        If True, compute the objective function at each step of the model.
        Default is False.

    threshold_lambda : float, optional
        threshold for removing (pruning) weights with high precision from
        the computation. Default is 1.e+4.

    fit_intercept : boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).
        Default is True.

    normalize : boolean, optional, default False
        If True, the regressors X will be normalized before regression.

    copy_X : boolean, optional, default True.
        If True, X will be copied; else, it may be overwritten.

    verbose : boolean, optional, default False
        Verbose mode when fitting the model.

    Attributes
    ----------
    coef_ : array, shape = (n_features)
        Coefficients of the regression model (mean of distribution)

    alpha_ : float
       estimated precision of the noise.

    lambda_ : array, shape = (n_features)
       estimated precisions of the weights.

    sigma_ : array, shape = (n_features, n_features)
        estimated variance-covariance matrix of the weights

    scores_ : float
        if computed, value of the objective function (to be maximized)

    Examples
    --------
    >>> from sklearn import linear_model
    >>> clf = linear_model.ARDRegression()
    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
    ... # doctest: +NORMALIZE_WHITESPACE
    ARDRegression(alpha_1=1e-06, alpha_2=1e-06, compute_score=False,
            copy_X=True, fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06,
            n_iter=300, normalize=False, threshold_lambda=10000.0, tol=0.001,
            verbose=False)
    >>> clf.predict([[1, 1]])
    array([ 1.])

    Notes
    --------
    See examples/linear_model/plot_ard.py for an example.
    """

    def __init__(self, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
                 lambda_1=1.e-6, lambda_2=1.e-6, compute_score=False,
                 threshold_lambda=1.e+4, fit_intercept=True, normalize=False,
                 copy_X=True, verbose=False):
        self.n_iter = n_iter
        self.tol = tol
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.alpha_1 = alpha_1
        self.alpha_2 = alpha_2
        self.lambda_1 = lambda_1
        self.lambda_2 = lambda_2
        self.compute_score = compute_score
        self.threshold_lambda = threshold_lambda
        self.copy_X = copy_X
        self.verbose = verbose

    def fit(self, X, y):
        """Fit the ARDRegression model according to the given training data
        and parameters.

        Iterative procedure to maximize the evidence

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.
        y : array, shape = [n_samples]
            Target values (integers)

        Returns
        -------
        self : returns an instance of self.
        """
        X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)

        n_samples, n_features = X.shape
        coef_ = np.zeros(n_features)

        X, y, X_mean, y_mean, X_std = self._center_data(
            X, y, self.fit_intercept, self.normalize, self.copy_X)

        ### Launch the convergence loop
        keep_lambda = np.ones(n_features, dtype=bool)

        lambda_1 = self.lambda_1
        lambda_2 = self.lambda_2
        alpha_1 = self.alpha_1
        alpha_2 = self.alpha_2
        verbose = self.verbose

        ### Initialization of the values of the parameters
        alpha_ = 1. / np.var(y)
        lambda_ = np.ones(n_features)

        self.scores_ = list()
        coef_old_ = None

        ### Iterative procedure of ARDRegression
        for iter_ in range(self.n_iter):
            ### Compute mu and sigma (using Woodbury matrix identity)
            sigma_ = pinvh(np.eye(n_samples) / alpha_ +
                           np.dot(X[:, keep_lambda] *
                           np.reshape(1. / lambda_[keep_lambda], [1, -1]),
                           X[:, keep_lambda].T))
            sigma_ = np.dot(sigma_, X[:, keep_lambda]
                            * np.reshape(1. / lambda_[keep_lambda], [1, -1]))
            sigma_ = - np.dot(np.reshape(1. / lambda_[keep_lambda], [-1, 1])
                              * X[:, keep_lambda].T, sigma_)
            sigma_.flat[::(sigma_.shape[1] + 1)] += 1. / lambda_[keep_lambda]
            coef_[keep_lambda] = alpha_ * np.dot(
                sigma_, np.dot(X[:, keep_lambda].T, y))

            ### Update alpha and lambda
            rmse_ = np.sum((y - np.dot(X, coef_)) ** 2)
            gamma_ = 1. - lambda_[keep_lambda] * np.diag(sigma_)
            lambda_[keep_lambda] = ((gamma_ + 2. * lambda_1)
                                    / ((coef_[keep_lambda]) ** 2
                                       + 2. * lambda_2))
            alpha_ = ((n_samples - gamma_.sum() + 2. * alpha_1)
                      / (rmse_ + 2. * alpha_2))

            ### Prune the weights with a precision over a threshold
            keep_lambda = lambda_ < self.threshold_lambda
            coef_[~keep_lambda] = 0

            ### Compute the objective function
            if self.compute_score:
                s = (lambda_1 * np.log(lambda_) - lambda_2 * lambda_).sum()
                s += alpha_1 * log(alpha_) - alpha_2 * alpha_
                s += 0.5 * (fast_logdet(sigma_) + n_samples * log(alpha_)
                                                + np.sum(np.log(lambda_)))
                s -= 0.5 * (alpha_ * rmse_ + (lambda_ * coef_ ** 2).sum())
                self.scores_.append(s)

            ### Check for convergence
            if iter_ > 0 and np.sum(np.abs(coef_old_ - coef_)) < self.tol:
                if verbose:
                    print("Converged after %s iterations" % iter_)
                break
            coef_old_ = np.copy(coef_)

        self.coef_ = coef_
        self.alpha_ = alpha_
        self.sigma_ = sigma_
        self.lambda_ = lambda_
        self._set_intercept(X_mean, y_mean, X_std)
        return self