# -*- coding: utf-8 -*-
import collections
from datetime import datetime
import re

import nose
from nose.tools import assert_equal, assert_true
import numpy as np
import pandas as pd
from pandas.tslib import iNaT, NaT
from pandas import Series, DataFrame, date_range, DatetimeIndex, Timestamp, Float64Index
from pandas import compat
from pandas.compat import range, long, lrange, lmap, u
from pandas.core.common import notnull, isnull, array_equivalent
import pandas.core.common as com
import pandas.core.convert as convert
import pandas.core.format as fmt
import pandas.util.testing as tm
import pandas.core.config as cf

_multiprocess_can_split_ = True

def test_mut_exclusive():
    msg = "mutually exclusive arguments: '[ab]' and '[ab]'"
    with tm.assertRaisesRegexp(TypeError, msg):
        com._mut_exclusive(a=1, b=2)
    assert com._mut_exclusive(a=1, b=None) == 1
    assert com._mut_exclusive(major=None, major_axis=None) is None

def test_is_sequence():
    is_seq = com.is_sequence
    assert(is_seq((1, 2)))
    assert(is_seq([1, 2]))
    assert(not is_seq("abcd"))
    assert(not is_seq(u("abcd")))
    assert(not is_seq(np.int64))

    class A(object):
        def __getitem__(self):
            return 1

    assert(not is_seq(A()))

def test_get_callable_name():
    from functools import partial
    getname = com._get_callable_name

    def fn(x):
        return x
    lambda_ = lambda x: x
    part1 = partial(fn)
    part2 = partial(part1)

    class somecall(object):
        def __call__(self):
            return x

    assert getname(fn) == 'fn'
    assert getname(lambda_)
    assert getname(part1) == 'fn'
    assert getname(part2) == 'fn'
    assert getname(somecall()) == 'somecall'
    assert getname(1) is None

#Issue 10859
class TestABCClasses(tm.TestCase):
    tuples = [[1, 2, 2], ['red', 'blue', 'red']]
    multi_index = pd.MultiIndex.from_arrays(tuples, names=('number', 'color'))
    datetime_index = pd.to_datetime(['2000/1/1', '2010/1/1'])
    timedelta_index = pd.to_timedelta(np.arange(5), unit='s')
    period_index = pd.period_range('2000/1/1', '2010/1/1/', freq='M')
    categorical = pd.Categorical([1, 2, 3], categories=[2, 3, 1])
    categorical_df = pd.DataFrame({"values": [1, 2, 3]}, index=categorical)
    df = pd.DataFrame({'names': ['a', 'b', 'c']}, index=multi_index)
    sparse_series = pd.Series([1, 2, 3]).to_sparse()
    sparse_array = pd.SparseArray(np.random.randn(10))

    def test_abc_types(self):
        self.assertIsInstance(pd.Index(['a', 'b', 'c']), com.ABCIndex)
        self.assertIsInstance(pd.Int64Index([1, 2, 3]), com.ABCInt64Index)
        self.assertIsInstance(pd.Float64Index([1, 2, 3]), com.ABCFloat64Index)
        self.assertIsInstance(self.multi_index, com.ABCMultiIndex)
        self.assertIsInstance(self.datetime_index, com.ABCDatetimeIndex)
        self.assertIsInstance(self.timedelta_index, com.ABCTimedeltaIndex)
        self.assertIsInstance(self.period_index, com.ABCPeriodIndex)
        self.assertIsInstance(self.categorical_df.index, com.ABCCategoricalIndex)
        self.assertIsInstance(pd.Index(['a', 'b', 'c']), com.ABCIndexClass)
        self.assertIsInstance(pd.Int64Index([1, 2, 3]), com.ABCIndexClass)
        self.assertIsInstance(pd.Series([1, 2, 3]), com.ABCSeries)
        self.assertIsInstance(self.df, com.ABCDataFrame)
        self.assertIsInstance(self.df.to_panel(), com.ABCPanel)
        self.assertIsInstance(self.sparse_series, com.ABCSparseSeries)
        self.assertIsInstance(self.sparse_array, com.ABCSparseArray)
        self.assertIsInstance(self.categorical, com.ABCCategorical)
        self.assertIsInstance(pd.Period('2012', freq='A-DEC'), com.ABCPeriod)

class TestInferDtype(tm.TestCase):

    def test_infer_dtype_from_scalar(self):
        # Test that _infer_dtype_from_scalar is returning correct dtype for int and float.

        for dtypec in [ np.uint8, np.int8,
                        np.uint16, np.int16,
                        np.uint32, np.int32,
                        np.uint64, np.int64 ]:
            data = dtypec(12)
            dtype, val = com._infer_dtype_from_scalar(data)
            self.assertEqual(dtype, type(data))

        data = 12
        dtype, val = com._infer_dtype_from_scalar(data)
        self.assertEqual(dtype, np.int64)

        for dtypec in [ np.float16, np.float32, np.float64 ]:
            data = dtypec(12)
            dtype, val = com._infer_dtype_from_scalar(data)
            self.assertEqual(dtype, dtypec)

        data = np.float(12)
        dtype, val = com._infer_dtype_from_scalar(data)
        self.assertEqual(dtype, np.float64)

        for data in [ True, False ]:
            dtype, val = com._infer_dtype_from_scalar(data)
            self.assertEqual(dtype, np.bool_)

        for data in [ np.complex64(1), np.complex128(1) ]:
            dtype, val = com._infer_dtype_from_scalar(data)
            self.assertEqual(dtype, np.complex_)

        import datetime
        for data in [ np.datetime64(1,'ns'),
                      pd.Timestamp(1),
                      datetime.datetime(2000,1,1,0,0)
                      ]:
            dtype, val = com._infer_dtype_from_scalar(data)
            self.assertEqual(dtype, 'M8[ns]')

        for data in [ np.timedelta64(1,'ns'),
                      pd.Timedelta(1),
                      datetime.timedelta(1)
                      ]:
            dtype, val = com._infer_dtype_from_scalar(data)
            self.assertEqual(dtype, 'm8[ns]')

        for data in [ datetime.date(2000,1,1),
                      pd.Timestamp(1,tz='US/Eastern'),
                      'foo'
                      ]:
            dtype, val = com._infer_dtype_from_scalar(data)
            self.assertEqual(dtype, np.object_)

def test_notnull():
    assert notnull(1.)
    assert not notnull(None)
    assert not notnull(np.NaN)

    with cf.option_context("mode.use_inf_as_null", False):
        assert notnull(np.inf)
        assert notnull(-np.inf)

        arr = np.array([1.5, np.inf, 3.5, -np.inf])
        result = notnull(arr)
        assert result.all()

    with cf.option_context("mode.use_inf_as_null", True):
        assert not notnull(np.inf)
        assert not notnull(-np.inf)

        arr = np.array([1.5, np.inf, 3.5, -np.inf])
        result = notnull(arr)
        assert result.sum() == 2

    with cf.option_context("mode.use_inf_as_null", False):
        for s in [tm.makeFloatSeries(),tm.makeStringSeries(),
                  tm.makeObjectSeries(),tm.makeTimeSeries(),tm.makePeriodSeries()]:
            assert(isinstance(isnull(s), Series))

def test_isnull():
    assert not isnull(1.)
    assert isnull(None)
    assert isnull(np.NaN)
    assert not isnull(np.inf)
    assert not isnull(-np.inf)

    # series
    for s in [tm.makeFloatSeries(),tm.makeStringSeries(),
              tm.makeObjectSeries(),tm.makeTimeSeries(),tm.makePeriodSeries()]:
        assert(isinstance(isnull(s), Series))

    # frame
    for df in [tm.makeTimeDataFrame(),tm.makePeriodFrame(),tm.makeMixedDataFrame()]:
        result = isnull(df)
        expected = df.apply(isnull)
        tm.assert_frame_equal(result, expected)

    # panel
    for p in [ tm.makePanel(), tm.makePeriodPanel(), tm.add_nans(tm.makePanel()) ]:
        result = isnull(p)
        expected = p.apply(isnull)
        tm.assert_panel_equal(result, expected)

    # panel 4d
    for p in [ tm.makePanel4D(), tm.add_nans_panel4d(tm.makePanel4D()) ]:
        result = isnull(p)
        expected = p.apply(isnull)
        tm.assert_panel4d_equal(result, expected)

def test_isnull_lists():
    result = isnull([[False]])
    exp = np.array([[False]])
    assert(np.array_equal(result, exp))

    result = isnull([[1], [2]])
    exp = np.array([[False], [False]])
    assert(np.array_equal(result, exp))

    # list of strings / unicode
    result = isnull(['foo', 'bar'])
    assert(not result.any())

    result = isnull([u('foo'), u('bar')])
    assert(not result.any())

def test_isnull_nat():
    result = isnull([NaT])
    exp = np.array([True])
    assert(np.array_equal(result, exp))

    result = isnull(np.array([NaT], dtype=object))
    exp = np.array([True])
    assert(np.array_equal(result, exp))

def test_isnull_numpy_nat():
    arr = np.array([NaT, np.datetime64('NaT'), np.timedelta64('NaT'),
                    np.datetime64('NaT', 's')])
    result = isnull(arr)
    expected = np.array([True] * 4)
    tm.assert_numpy_array_equal(result, expected)

def test_isnull_datetime():
    assert (not isnull(datetime.now()))
    assert notnull(datetime.now())

    idx = date_range('1/1/1990', periods=20)
    assert(notnull(idx).all())

    idx = np.asarray(idx)
    idx[0] = iNaT
    idx = DatetimeIndex(idx)
    mask = isnull(idx)
    assert(mask[0])
    assert(not mask[1:].any())

    # GH 9129
    pidx = idx.to_period(freq='M')
    mask = isnull(pidx)
    assert(mask[0])
    assert(not mask[1:].any())

    mask = isnull(pidx[1:])
    assert(not mask.any())

class TestIsNull(tm.TestCase):
    def test_0d_array(self):
        self.assertTrue(isnull(np.array(np.nan)))
        self.assertFalse(isnull(np.array(0.0)))
        self.assertFalse(isnull(np.array(0)))
        # test object dtype
        self.assertTrue(isnull(np.array(np.nan, dtype=object)))
        self.assertFalse(isnull(np.array(0.0, dtype=object)))
        self.assertFalse(isnull(np.array(0, dtype=object)))

def test_downcast_conv():
    # test downcasting

    arr = np.array([8.5, 8.6, 8.7, 8.8, 8.9999999999995])
    result = com._possibly_downcast_to_dtype(arr, 'infer')
    assert (np.array_equal(result, arr))

    arr = np.array([8., 8., 8., 8., 8.9999999999995])
    result = com._possibly_downcast_to_dtype(arr, 'infer')
    expected = np.array([8, 8, 8, 8, 9])
    assert (np.array_equal(result, expected))

    arr = np.array([8., 8., 8., 8., 9.0000000000005])
    result = com._possibly_downcast_to_dtype(arr, 'infer')
    expected = np.array([8, 8, 8, 8, 9])
    assert (np.array_equal(result, expected))

    # conversions

    expected = np.array([1,2])
    for dtype in [np.float64,object,np.int64]:
        arr = np.array([1.0,2.0],dtype=dtype)
        result = com._possibly_downcast_to_dtype(arr,'infer')
        tm.assert_almost_equal(result, expected)

    expected = np.array([1.0,2.0,np.nan])
    for dtype in [np.float64,object]:
        arr = np.array([1.0,2.0,np.nan],dtype=dtype)
        result = com._possibly_downcast_to_dtype(arr,'infer')
        tm.assert_almost_equal(result, expected)

    # empties
    for dtype in [np.int32,np.float64,np.float32,np.bool_,np.int64,object]:
        arr = np.array([],dtype=dtype)
        result = com._possibly_downcast_to_dtype(arr,'int64')
        tm.assert_almost_equal(result, np.array([],dtype=np.int64))
        assert result.dtype == np.int64

def test_array_equivalent():
    assert array_equivalent(np.array([np.nan, np.nan]),
                            np.array([np.nan, np.nan]))
    assert array_equivalent(np.array([np.nan, 1, np.nan]),
                            np.array([np.nan, 1, np.nan]))
    assert array_equivalent(np.array([np.nan, None], dtype='object'),
                            np.array([np.nan, None], dtype='object'))
    assert array_equivalent(np.array([np.nan, 1+1j], dtype='complex'),
                            np.array([np.nan, 1+1j], dtype='complex'))
    assert not array_equivalent(np.array([np.nan, 1+1j], dtype='complex'),
                                np.array([np.nan, 1+2j], dtype='complex'))
    assert not array_equivalent(np.array([np.nan, 1, np.nan]),
                                np.array([np.nan, 2, np.nan]))
    assert not array_equivalent(np.array(['a', 'b', 'c', 'd']), np.array(['e', 'e']))
    assert array_equivalent(Float64Index([0, np.nan]), Float64Index([0, np.nan]))
    assert not array_equivalent(Float64Index([0, np.nan]), Float64Index([1, np.nan]))
    assert array_equivalent(DatetimeIndex([0, np.nan]), DatetimeIndex([0, np.nan]))
    assert not array_equivalent(DatetimeIndex([0, np.nan]), DatetimeIndex([1, np.nan]))

def test_datetimeindex_from_empty_datetime64_array():
    for unit in [ 'ms', 'us', 'ns' ]:
        idx = DatetimeIndex(np.array([], dtype='datetime64[%s]' % unit))
        assert(len(idx) == 0)

def test_nan_to_nat_conversions():

    df = DataFrame(dict({
        'A' : np.asarray(lrange(10),dtype='float64'),
        'B' : Timestamp('20010101') }))
    df.iloc[3:6,:] = np.nan
    result = df.loc[4,'B'].value
    assert(result == iNaT)

    s = df['B'].copy()
    s._data = s._data.setitem(indexer=tuple([slice(8,9)]),value=np.nan)
    assert(isnull(s[8]))

    # numpy < 1.7.0 is wrong
    from distutils.version import LooseVersion
    if LooseVersion(np.__version__) >= '1.7.0':
        assert(s[8].value == np.datetime64('NaT').astype(np.int64))

def test_any_none():
    assert(com._any_none(1, 2, 3, None))
    assert(not com._any_none(1, 2, 3, 4))

def test_all_not_none():
    assert(com._all_not_none(1, 2, 3, 4))
    assert(not com._all_not_none(1, 2, 3, None))
    assert(not com._all_not_none(None, None, None, None))

def test_repr_binary_type():
    import string
    letters = string.ascii_letters
    btype = compat.binary_type
    try:
        raw = btype(letters, encoding=cf.get_option('display.encoding'))
    except TypeError:
        raw = btype(letters)
    b = compat.text_type(compat.bytes_to_str(raw))
    res = com.pprint_thing(b, quote_strings=True)
    assert_equal(res, repr(b))
    res = com.pprint_thing(b, quote_strings=False)
    assert_equal(res, b)

def test_adjoin():
    data = [['a', 'b', 'c'],
            ['dd', 'ee', 'ff'],
            ['ggg', 'hhh', 'iii']]
    expected = 'a  dd  ggg\nb  ee  hhh\nc  ff  iii'

    adjoined = com.adjoin(2, *data)

    assert(adjoined == expected)

class TestFormattBase(tm.TestCase):

    def test_adjoin(self):
        data = [['a', 'b', 'c'],
                ['dd', 'ee', 'ff'],
                ['ggg', 'hhh', 'iii']]
        expected = 'a  dd  ggg\nb  ee  hhh\nc  ff  iii'

        adjoined = com.adjoin(2, *data)

        self.assertEqual(adjoined, expected)

    def test_adjoin_unicode(self):
        data = [[u'あ', 'b', 'c'],
                ['dd', u'ええ', 'ff'],
                ['ggg', 'hhh', u'いいい']]
        expected = u'あ  dd  ggg\nb  ええ  hhh\nc  ff  いいい'
        adjoined = com.adjoin(2, *data)
        self.assertEqual(adjoined, expected)

        adj = fmt.EastAsianTextAdjustment()

        expected = u"""あ  dd    ggg
b   ええ  hhh
c   ff    いいい"""
        adjoined = adj.adjoin(2, *data)
        self.assertEqual(adjoined, expected)
        cols = adjoined.split('\n')
        self.assertEqual(adj.len(cols[0]), 13)
        self.assertEqual(adj.len(cols[1]), 13)
        self.assertEqual(adj.len(cols[2]), 16)

        expected = u"""あ       dd         ggg
b        ええ       hhh
c        ff         いいい"""
        adjoined = adj.adjoin(7, *data)
        self.assertEqual(adjoined, expected)
        cols = adjoined.split('\n')
        self.assertEqual(adj.len(cols[0]), 23)
        self.assertEqual(adj.len(cols[1]), 23)
        self.assertEqual(adj.len(cols[2]), 26)

    def test_justify(self):
        adj = fmt.EastAsianTextAdjustment()

        def just(x, *args, **kwargs):
            # wrapper to test single str
            return adj.justify([x], *args, **kwargs)[0]

        self.assertEqual(just('abc', 5, mode='left'), 'abc  ')
        self.assertEqual(just('abc', 5, mode='center'), ' abc ')
        self.assertEqual(just('abc', 5, mode='right'), '  abc')
        self.assertEqual(just(u'abc', 5, mode='left'), 'abc  ')
        self.assertEqual(just(u'abc', 5, mode='center'), ' abc ')
        self.assertEqual(just(u'abc', 5, mode='right'), '  abc')

        self.assertEqual(just(u'パンダ', 5, mode='left'), u'パンダ')
        self.assertEqual(just(u'パンダ', 5, mode='center'), u'パンダ')
        self.assertEqual(just(u'パンダ', 5, mode='right'), u'パンダ')

        self.assertEqual(just(u'パンダ', 10, mode='left'), u'パンダ    ')
        self.assertEqual(just(u'パンダ', 10, mode='center'), u'  パンダ  ')
        self.assertEqual(just(u'パンダ', 10, mode='right'), u'    パンダ')

    def test_east_asian_len(self):
        adj = fmt.EastAsianTextAdjustment()

        self.assertEqual(adj.len('abc'), 3)
        self.assertEqual(adj.len(u'abc'), 3)

        self.assertEqual(adj.len(u'パンダ'), 6)
        self.assertEqual(adj.len(u'ﾊﾟﾝﾀﾞ'), 5)
        self.assertEqual(adj.len(u'パンダpanda'), 11)
        self.assertEqual(adj.len(u'ﾊﾟﾝﾀﾞpanda'), 10)

    def test_ambiguous_width(self):
        adj = fmt.EastAsianTextAdjustment()
        self.assertEqual(adj.len(u'¡¡ab'), 4)

        with cf.option_context('display.unicode.ambiguous_as_wide', True):
            adj = fmt.EastAsianTextAdjustment()
            self.assertEqual(adj.len(u'¡¡ab'), 6)

        data = [[u'あ', 'b', 'c'],
                ['dd', u'ええ', 'ff'],
                ['ggg', u'¡¡ab', u'いいい']]
        expected = u'あ  dd    ggg \nb   ええ  ¡¡ab\nc   ff    いいい'
        adjoined = adj.adjoin(2, *data)
        self.assertEqual(adjoined, expected)

def test_iterpairs():
    data = [1, 2, 3, 4]
    expected = [(1, 2),
                (2, 3),
                (3, 4)]

    result = list(com.iterpairs(data))

    assert(result == expected)

def test_split_ranges():
    def _bin(x, width):
        "return int(x) as a base2 string of given width"
        return ''.join(str((x >> i) & 1) for i in range(width - 1, -1, -1))

    def test_locs(mask):
        nfalse = sum(np.array(mask) == 0)

        remaining = 0
        for s, e in com.split_ranges(mask):
            remaining += e - s

            assert 0 not in mask[s:e]

        # make sure the total items covered by the ranges are a complete cover
        assert remaining + nfalse == len(mask)

    # exhaustively test all possible mask sequences of length 8
    ncols = 8
    for i in range(2 ** ncols):
        cols = lmap(int, list(_bin(i, ncols)))  # count up in base2
        mask = [cols[i] == 1 for i in range(len(cols))]
        test_locs(mask)

    # base cases
    test_locs([])
    test_locs([0])
    test_locs([1])

def test_indent():
    s = 'a b c\nd e f'
    result = com.indent(s, spaces=6)

    assert(result == '      a b c\n      d e f')

def test_banner():
    ban = com.banner('hi')
    assert(ban == ('%s\nhi\n%s' % ('=' * 80, '=' * 80)))

def test_map_indices_py():
    data = [4, 3, 2, 1]
    expected = {4: 0, 3: 1, 2: 2, 1: 3}

    result = com.map_indices_py(data)

    assert(result == expected)

def test_union():
    a = [1, 2, 3]
    b = [4, 5, 6]

    union = sorted(com.union(a, b))

    assert((a + b) == union)

def test_difference():
    a = [1, 2, 3]
    b = [1, 2, 3, 4, 5, 6]

    inter = sorted(com.difference(b, a))

    assert([4, 5, 6] == inter)

def test_intersection():
    a = [1, 2, 3]
    b = [1, 2, 3, 4, 5, 6]

    inter = sorted(com.intersection(a, b))

    assert(a == inter)

def test_groupby():
    values = ['foo', 'bar', 'baz', 'baz2', 'qux', 'foo3']
    expected = {'f': ['foo', 'foo3'],
                'b': ['bar', 'baz', 'baz2'],
                'q': ['qux']}

    grouped = com.groupby(values, lambda x: x[0])

    for k, v in grouped:
        assert v == expected[k]

def test_is_list_like():
    passes = ([], [1], (1,), (1, 2), {'a': 1}, set([1, 'a']), Series([1]),
              Series([]), Series(['a']).str)
    fails = (1, '2', object())

    for p in passes:
        assert com.is_list_like(p)

    for f in fails:
        assert not com.is_list_like(f)

def test_is_named_tuple():
    passes = (collections.namedtuple('Test',list('abc'))(1,2,3),)
    fails = ((1,2,3), 'a', Series({'pi':3.14}))

    for p in passes:
        assert com.is_named_tuple(p)

    for f in fails:
        assert not com.is_named_tuple(f)

def test_is_hashable():

    # all new-style classes are hashable by default
    class HashableClass(object):
        pass

    class UnhashableClass1(object):
        __hash__ = None

    class UnhashableClass2(object):
        def __hash__(self):
            raise TypeError("Not hashable")

    hashable = (
        1, 3.14, np.float64(3.14), 'a', tuple(), (1,), HashableClass(),
    )
    not_hashable = (
        [], UnhashableClass1(),
    )
    abc_hashable_not_really_hashable = (
        ([],), UnhashableClass2(),
    )

    for i in hashable:
        assert com.is_hashable(i)
    for i in not_hashable:
        assert not com.is_hashable(i)
    for i in abc_hashable_not_really_hashable:
        assert not com.is_hashable(i)

    # numpy.array is no longer collections.Hashable as of
    # https://github.com/numpy/numpy/pull/5326, just test
    # pandas.common.is_hashable()
    assert not com.is_hashable(np.array([]))

    # old-style classes in Python 2 don't appear hashable to
    # collections.Hashable but also seem to support hash() by default
    if compat.PY2:
        class OldStyleClass():
            pass
        c = OldStyleClass()
        assert not isinstance(c, collections.Hashable)
        assert com.is_hashable(c)
        hash(c)  # this will not raise

def test_ensure_int32():
    values = np.arange(10, dtype=np.int32)
    result = com._ensure_int32(values)
    assert(result.dtype == np.int32)

    values = np.arange(10, dtype=np.int64)
    result = com._ensure_int32(values)
    assert(result.dtype == np.int32)

def test_ensure_platform_int():

    # verify that when we create certain types of indices
    # they remain the correct type under platform conversions
    from pandas.core.index import Int64Index

    # int64
    x = Int64Index([1, 2, 3], dtype='int64')
    assert(x.dtype == np.int64)

    pi = com._ensure_platform_int(x)
    assert(pi.dtype == np.int_)

    # int32
    x = Int64Index([1, 2, 3], dtype='int32')
    assert(x.dtype == np.int32)

    pi = com._ensure_platform_int(x)
    assert(pi.dtype == np.int_)

# TODO: fix this broken test

# def test_console_encode():
#     """
#     On Python 2, if sys.stdin.encoding is None (IPython with zmq frontend)
#     common.console_encode should encode things as utf-8.
#     """
#     if compat.PY3:
#         raise nose.SkipTest

#     with tm.stdin_encoding(encoding=None):
#         result = com.console_encode(u"\u05d0")
#         expected = u"\u05d0".encode('utf-8')
#         assert (result == expected)

def test_is_re():
    passes = re.compile('ad'),
    fails = 'x', 2, 3, object()

    for p in passes:
        assert com.is_re(p)

    for f in fails:
        assert not com.is_re(f)

def test_is_recompilable():
    passes = (r'a', u('x'), r'asdf', re.compile('adsf'),
              u(r'\u2233\s*'), re.compile(r''))
    fails = 1, [], object()

    for p in passes:
        assert com.is_re_compilable(p)

    for f in fails:
        assert not com.is_re_compilable(f)

def test_random_state():
    import numpy.random as npr
    # Check with seed
    state = com._random_state(5)
    assert_equal(state.uniform(), npr.RandomState(5).uniform())

    # Check with random state object
    state2 = npr.RandomState(10)
    assert_equal(com._random_state(state2).uniform(), npr.RandomState(10).uniform())

    # check with no arg random state
    assert isinstance(com._random_state(), npr.RandomState)

    # Error for floats or strings
    with tm.assertRaises(ValueError):
        com._random_state('test')

    with tm.assertRaises(ValueError):
        com._random_state(5.5)

def test_maybe_match_name():

    matched = com._maybe_match_name(Series([1], name='x'), Series([2], name='x'))
    assert(matched == 'x')

    matched = com._maybe_match_name(Series([1], name='x'), Series([2], name='y'))
    assert(matched is None)

    matched = com._maybe_match_name(Series([1]), Series([2], name='x'))
    assert(matched is None)

    matched = com._maybe_match_name(Series([1], name='x'), Series([2]))
    assert(matched is None)

    matched = com._maybe_match_name(Series([1], name='x'), [2])
    assert(matched == 'x')

    matched = com._maybe_match_name([1], Series([2], name='y'))
    assert(matched == 'y')

class TestTake(tm.TestCase):
    # standard incompatible fill error
    fill_error = re.compile("Incompatible type for fill_value")

    _multiprocess_can_split_ = True

    def test_1d_with_out(self):
        def _test_dtype(dtype, can_hold_na):
            data = np.random.randint(0, 2, 4).astype(dtype)

            indexer = [2, 1, 0, 1]
            out = np.empty(4, dtype=dtype)
            com.take_1d(data, indexer, out=out)
            expected = data.take(indexer)
            tm.assert_almost_equal(out, expected)

            indexer = [2, 1, 0, -1]
            out = np.empty(4, dtype=dtype)
            if can_hold_na:
                com.take_1d(data, indexer, out=out)
                expected = data.take(indexer)
                expected[3] = np.nan
                tm.assert_almost_equal(out, expected)
            else:
                with tm.assertRaisesRegexp(TypeError, self.fill_error):
                    com.take_1d(data, indexer, out=out)
                # no exception o/w
                data.take(indexer, out=out)

        _test_dtype(np.float64, True)
        _test_dtype(np.float32, True)
        _test_dtype(np.uint64, False)
        _test_dtype(np.uint32, False)
        _test_dtype(np.uint16, False)
        _test_dtype(np.uint8, False)
        _test_dtype(np.int64, False)
        _test_dtype(np.int32, False)
        _test_dtype(np.int16, False)
        _test_dtype(np.int8, False)
        _test_dtype(np.object_, True)
        _test_dtype(np.bool, False)

    def test_1d_fill_nonna(self):
        def _test_dtype(dtype, fill_value, out_dtype):
            data = np.random.randint(0, 2, 4).astype(dtype)

            indexer = [2, 1, 0, -1]

            result = com.take_1d(data, indexer, fill_value=fill_value)
            assert((result[[0, 1, 2]] == data[[2, 1, 0]]).all())
            assert(result[3] == fill_value)
            assert(result.dtype == out_dtype)

            indexer = [2, 1, 0, 1]

            result = com.take_1d(data, indexer, fill_value=fill_value)
            assert((result[[0, 1, 2, 3]] == data[indexer]).all())
            assert(result.dtype == dtype)

        _test_dtype(np.int8, np.int16(127), np.int8)
        _test_dtype(np.int8, np.int16(128), np.int16)
        _test_dtype(np.int32, 1, np.int32)
        _test_dtype(np.int32, 2.0, np.float64)
        _test_dtype(np.int32, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.int32, True, np.object_)
        _test_dtype(np.int32, '', np.object_)
        _test_dtype(np.float64, 1, np.float64)
        _test_dtype(np.float64, 2.0, np.float64)
        _test_dtype(np.float64, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.float64, True, np.object_)
        _test_dtype(np.float64, '', np.object_)
        _test_dtype(np.complex128, 1, np.complex128)
        _test_dtype(np.complex128, 2.0, np.complex128)
        _test_dtype(np.complex128, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.complex128, True, np.object_)
        _test_dtype(np.complex128, '', np.object_)
        _test_dtype(np.bool_, 1, np.object_)
        _test_dtype(np.bool_, 2.0, np.object_)
        _test_dtype(np.bool_, 3.0 + 4.0j, np.object_)
        _test_dtype(np.bool_, True, np.bool_)
        _test_dtype(np.bool_, '', np.object_)

    def test_2d_with_out(self):
        def _test_dtype(dtype, can_hold_na, writeable=True):
            data = np.random.randint(0, 2, (5, 3)).astype(dtype)
            data.flags.writeable = writeable

            indexer = [2, 1, 0, 1]
            out0 = np.empty((4, 3), dtype=dtype)
            out1 = np.empty((5, 4), dtype=dtype)
            com.take_nd(data, indexer, out=out0, axis=0)
            com.take_nd(data, indexer, out=out1, axis=1)
            expected0 = data.take(indexer, axis=0)
            expected1 = data.take(indexer, axis=1)
            tm.assert_almost_equal(out0, expected0)
            tm.assert_almost_equal(out1, expected1)

            indexer = [2, 1, 0, -1]
            out0 = np.empty((4, 3), dtype=dtype)
            out1 = np.empty((5, 4), dtype=dtype)
            if can_hold_na:
                com.take_nd(data, indexer, out=out0, axis=0)
                com.take_nd(data, indexer, out=out1, axis=1)
                expected0 = data.take(indexer, axis=0)
                expected1 = data.take(indexer, axis=1)
                expected0[3, :] = np.nan
                expected1[:, 3] = np.nan
                tm.assert_almost_equal(out0, expected0)
                tm.assert_almost_equal(out1, expected1)
            else:
                for i, out in enumerate([out0, out1]):
                    with tm.assertRaisesRegexp(TypeError, self.fill_error):
                        com.take_nd(data, indexer, out=out, axis=i)
                    # no exception o/w
                    data.take(indexer, out=out, axis=i)

        for writeable in [True, False]:
            # Check that take_nd works both with writeable arrays (in which
            # case fast typed memoryviews implementation) and read-only
            # arrays alike.
            _test_dtype(np.float64, True, writeable=writeable)
            _test_dtype(np.float32, True, writeable=writeable)
            _test_dtype(np.uint64, False, writeable=writeable)
            _test_dtype(np.uint32, False, writeable=writeable)
            _test_dtype(np.uint16, False, writeable=writeable)
            _test_dtype(np.uint8, False, writeable=writeable)
            _test_dtype(np.int64, False, writeable=writeable)
            _test_dtype(np.int32, False, writeable=writeable)
            _test_dtype(np.int16, False, writeable=writeable)
            _test_dtype(np.int8, False, writeable=writeable)
            _test_dtype(np.object_, True, writeable=writeable)
            _test_dtype(np.bool, False, writeable=writeable)

    def test_2d_fill_nonna(self):
        def _test_dtype(dtype, fill_value, out_dtype):
            data = np.random.randint(0, 2, (5, 3)).astype(dtype)

            indexer = [2, 1, 0, -1]

            result = com.take_nd(data, indexer, axis=0, fill_value=fill_value)
            assert((result[[0, 1, 2], :] == data[[2, 1, 0], :]).all())
            assert((result[3, :] == fill_value).all())
            assert(result.dtype == out_dtype)

            result = com.take_nd(data, indexer, axis=1, fill_value=fill_value)
            assert((result[:, [0, 1, 2]] == data[:, [2, 1, 0]]).all())
            assert((result[:, 3] == fill_value).all())
            assert(result.dtype == out_dtype)

            indexer = [2, 1, 0, 1]

            result = com.take_nd(data, indexer, axis=0, fill_value=fill_value)
            assert((result[[0, 1, 2, 3], :] == data[indexer, :]).all())
            assert(result.dtype == dtype)

            result = com.take_nd(data, indexer, axis=1, fill_value=fill_value)
            assert((result[:, [0, 1, 2, 3]] == data[:, indexer]).all())
            assert(result.dtype == dtype)

        _test_dtype(np.int8, np.int16(127), np.int8)
        _test_dtype(np.int8, np.int16(128), np.int16)
        _test_dtype(np.int32, 1, np.int32)
        _test_dtype(np.int32, 2.0, np.float64)
        _test_dtype(np.int32, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.int32, True, np.object_)
        _test_dtype(np.int32, '', np.object_)
        _test_dtype(np.float64, 1, np.float64)
        _test_dtype(np.float64, 2.0, np.float64)
        _test_dtype(np.float64, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.float64, True, np.object_)
        _test_dtype(np.float64, '', np.object_)
        _test_dtype(np.complex128, 1, np.complex128)
        _test_dtype(np.complex128, 2.0, np.complex128)
        _test_dtype(np.complex128, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.complex128, True, np.object_)
        _test_dtype(np.complex128, '', np.object_)
        _test_dtype(np.bool_, 1, np.object_)
        _test_dtype(np.bool_, 2.0, np.object_)
        _test_dtype(np.bool_, 3.0 + 4.0j, np.object_)
        _test_dtype(np.bool_, True, np.bool_)
        _test_dtype(np.bool_, '', np.object_)

    def test_3d_with_out(self):
        def _test_dtype(dtype, can_hold_na):
            data = np.random.randint(0, 2, (5, 4, 3)).astype(dtype)

            indexer = [2, 1, 0, 1]
            out0 = np.empty((4, 4, 3), dtype=dtype)
            out1 = np.empty((5, 4, 3), dtype=dtype)
            out2 = np.empty((5, 4, 4), dtype=dtype)
            com.take_nd(data, indexer, out=out0, axis=0)
            com.take_nd(data, indexer, out=out1, axis=1)
            com.take_nd(data, indexer, out=out2, axis=2)
            expected0 = data.take(indexer, axis=0)
            expected1 = data.take(indexer, axis=1)
            expected2 = data.take(indexer, axis=2)
            tm.assert_almost_equal(out0, expected0)
            tm.assert_almost_equal(out1, expected1)
            tm.assert_almost_equal(out2, expected2)

            indexer = [2, 1, 0, -1]
            out0 = np.empty((4, 4, 3), dtype=dtype)
            out1 = np.empty((5, 4, 3), dtype=dtype)
            out2 = np.empty((5, 4, 4), dtype=dtype)
            if can_hold_na:
                com.take_nd(data, indexer, out=out0, axis=0)
                com.take_nd(data, indexer, out=out1, axis=1)
                com.take_nd(data, indexer, out=out2, axis=2)
                expected0 = data.take(indexer, axis=0)
                expected1 = data.take(indexer, axis=1)
                expected2 = data.take(indexer, axis=2)
                expected0[3, :, :] = np.nan
                expected1[:, 3, :] = np.nan
                expected2[:, :, 3] = np.nan
                tm.assert_almost_equal(out0, expected0)
                tm.assert_almost_equal(out1, expected1)
                tm.assert_almost_equal(out2, expected2)
            else:
                for i, out in enumerate([out0, out1, out2]):
                    with tm.assertRaisesRegexp(TypeError, self.fill_error):
                        com.take_nd(data, indexer, out=out, axis=i)
                    # no exception o/w
                    data.take(indexer, out=out, axis=i)

        _test_dtype(np.float64, True)
        _test_dtype(np.float32, True)
        _test_dtype(np.uint64, False)
        _test_dtype(np.uint32, False)
        _test_dtype(np.uint16, False)
        _test_dtype(np.uint8, False)
        _test_dtype(np.int64, False)
        _test_dtype(np.int32, False)
        _test_dtype(np.int16, False)
        _test_dtype(np.int8, False)
        _test_dtype(np.object_, True)
        _test_dtype(np.bool, False)

    def test_3d_fill_nonna(self):
        def _test_dtype(dtype, fill_value, out_dtype):
            data = np.random.randint(0, 2, (5, 4, 3)).astype(dtype)

            indexer = [2, 1, 0, -1]

            result = com.take_nd(data, indexer, axis=0, fill_value=fill_value)
            assert((result[[0, 1, 2], :, :] == data[[2, 1, 0], :, :]).all())
            assert((result[3, :, :] == fill_value).all())
            assert(result.dtype == out_dtype)

            result = com.take_nd(data, indexer, axis=1, fill_value=fill_value)
            assert((result[:, [0, 1, 2], :] == data[:, [2, 1, 0], :]).all())
            assert((result[:, 3, :] == fill_value).all())
            assert(result.dtype == out_dtype)

            result = com.take_nd(data, indexer, axis=2, fill_value=fill_value)
            assert((result[:, :, [0, 1, 2]] == data[:, :, [2, 1, 0]]).all())
            assert((result[:, :, 3] == fill_value).all())
            assert(result.dtype == out_dtype)

            indexer = [2, 1, 0, 1]

            result = com.take_nd(data, indexer, axis=0, fill_value=fill_value)
            assert((result[[0, 1, 2, 3], :, :] == data[indexer, :, :]).all())
            assert(result.dtype == dtype)

            result = com.take_nd(data, indexer, axis=1, fill_value=fill_value)
            assert((result[:, [0, 1, 2, 3], :] == data[:, indexer, :]).all())
            assert(result.dtype == dtype)

            result = com.take_nd(data, indexer, axis=2, fill_value=fill_value)
            assert((result[:, :, [0, 1, 2, 3]] == data[:, :, indexer]).all())
            assert(result.dtype == dtype)

        _test_dtype(np.int8, np.int16(127), np.int8)
        _test_dtype(np.int8, np.int16(128), np.int16)
        _test_dtype(np.int32, 1, np.int32)
        _test_dtype(np.int32, 2.0, np.float64)
        _test_dtype(np.int32, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.int32, True, np.object_)
        _test_dtype(np.int32, '', np.object_)
        _test_dtype(np.float64, 1, np.float64)
        _test_dtype(np.float64, 2.0, np.float64)
        _test_dtype(np.float64, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.float64, True, np.object_)
        _test_dtype(np.float64, '', np.object_)
        _test_dtype(np.complex128, 1, np.complex128)
        _test_dtype(np.complex128, 2.0, np.complex128)
        _test_dtype(np.complex128, 3.0 + 4.0j, np.complex128)
        _test_dtype(np.complex128, True, np.object_)
        _test_dtype(np.complex128, '', np.object_)
        _test_dtype(np.bool_, 1, np.object_)
        _test_dtype(np.bool_, 2.0, np.object_)
        _test_dtype(np.bool_, 3.0 + 4.0j, np.object_)
        _test_dtype(np.bool_, True, np.bool_)
        _test_dtype(np.bool_, '', np.object_)

    def test_1d_other_dtypes(self):
        arr = np.random.randn(10).astype(np.float32)

        indexer = [1, 2, 3, -1]
        result = com.take_1d(arr, indexer)
        expected = arr.take(indexer)
        expected[-1] = np.nan
        tm.assert_almost_equal(result, expected)

    def test_2d_other_dtypes(self):
        arr = np.random.randn(10, 5).astype(np.float32)

        indexer = [1, 2, 3, -1]

        # axis=0
        result = com.take_nd(arr, indexer, axis=0)
        expected = arr.take(indexer, axis=0)
        expected[-1] = np.nan
        tm.assert_almost_equal(result, expected)

        # axis=1
        result = com.take_nd(arr, indexer, axis=1)
        expected = arr.take(indexer, axis=1)
        expected[:, -1] = np.nan
        tm.assert_almost_equal(result, expected)

    def test_1d_bool(self):
        arr = np.array([0, 1, 0], dtype=bool)

        result = com.take_1d(arr, [0, 2, 2, 1])
        expected = arr.take([0, 2, 2, 1])
        self.assert_numpy_array_equal(result, expected)

        result = com.take_1d(arr, [0, 2, -1])
        self.assertEqual(result.dtype, np.object_)

    def test_2d_bool(self):
        arr = np.array([[0, 1, 0],
                        [1, 0, 1],
                        [0, 1, 1]], dtype=bool)

        result = com.take_nd(arr, [0, 2, 2, 1])
        expected = arr.take([0, 2, 2, 1], axis=0)
        self.assert_numpy_array_equal(result, expected)

        result = com.take_nd(arr, [0, 2, 2, 1], axis=1)
        expected = arr.take([0, 2, 2, 1], axis=1)
        self.assert_numpy_array_equal(result, expected)

        result = com.take_nd(arr, [0, 2, -1])
        self.assertEqual(result.dtype, np.object_)

    def test_2d_float32(self):
        arr = np.random.randn(4, 3).astype(np.float32)
        indexer = [0, 2, -1, 1, -1]

        # axis=0
        result = com.take_nd(arr, indexer, axis=0)
        result2 = np.empty_like(result)
        com.take_nd(arr, indexer, axis=0, out=result2)
        tm.assert_almost_equal(result, result2)

        expected = arr.take(indexer, axis=0)
        expected[[2, 4], :] = np.nan
        tm.assert_almost_equal(result, expected)

        #### this now accepts a float32! # test with float64 out buffer
        out = np.empty((len(indexer), arr.shape[1]), dtype='float32')
        com.take_nd(arr, indexer, out=out)  # it works!

        # axis=1
        result = com.take_nd(arr, indexer, axis=1)
        result2 = np.empty_like(result)
        com.take_nd(arr, indexer, axis=1, out=result2)
        tm.assert_almost_equal(result, result2)

        expected = arr.take(indexer, axis=1)
        expected[:, [2, 4]] = np.nan
        tm.assert_almost_equal(result, expected)

    def test_2d_datetime64(self):
        # 2005/01/01 - 2006/01/01
        arr = np.random.randint(long(11045376), long(11360736), (5, 3))*100000000000
        arr = arr.view(dtype='datetime64[ns]')
        indexer = [0, 2, -1, 1, -1]

        # axis=0
        result = com.take_nd(arr, indexer, axis=0)
        result2 = np.empty_like(result)
        com.take_nd(arr, indexer, axis=0, out=result2)
        tm.assert_almost_equal(result, result2)

        expected = arr.take(indexer, axis=0)
        expected.view(np.int64)[[2, 4], :] = iNaT
        tm.assert_almost_equal(result, expected)

        result = com.take_nd(arr, indexer, axis=0,
                             fill_value=datetime(2007, 1, 1))
        result2 = np.empty_like(result)
        com.take_nd(arr, indexer, out=result2, axis=0,
                    fill_value=datetime(2007, 1, 1))
        tm.assert_almost_equal(result, result2)

        expected = arr.take(indexer, axis=0)
        expected[[2, 4], :] = datetime(2007, 1, 1)
        tm.assert_almost_equal(result, expected)

        # axis=1
        result = com.take_nd(arr, indexer, axis=1)
        result2 = np.empty_like(result)
        com.take_nd(arr, indexer, axis=1, out=result2)
        tm.assert_almost_equal(result, result2)

        expected = arr.take(indexer, axis=1)
        expected.view(np.int64)[:, [2, 4]] = iNaT
        tm.assert_almost_equal(result, expected)

        result = com.take_nd(arr, indexer, axis=1,
                             fill_value=datetime(2007, 1, 1))
        result2 = np.empty_like(result)
        com.take_nd(arr, indexer, out=result2, axis=1,
                    fill_value=datetime(2007, 1, 1))
        tm.assert_almost_equal(result, result2)

        expected = arr.take(indexer, axis=1)
        expected[:, [2, 4]] = datetime(2007, 1, 1)
        tm.assert_almost_equal(result, expected)

class TestMaybe(tm.TestCase):

    def test_maybe_convert_string_to_array(self):
        result = com._maybe_convert_string_to_object('x')
        tm.assert_numpy_array_equal(result, np.array(['x'], dtype=object))
        self.assertTrue(result.dtype == object)

        result = com._maybe_convert_string_to_object(1)
        self.assertEqual(result, 1)

        arr = np.array(['x', 'y'], dtype=str)
        result = com._maybe_convert_string_to_object(arr)
        tm.assert_numpy_array_equal(result, np.array(['x', 'y'], dtype=object))
        self.assertTrue(result.dtype == object)

        # unicode
        arr = np.array(['x', 'y']).astype('U')
        result = com._maybe_convert_string_to_object(arr)
        tm.assert_numpy_array_equal(result, np.array(['x', 'y'], dtype=object))
        self.assertTrue(result.dtype == object)

        # object
        arr = np.array(['x', 2], dtype=object)
        result = com._maybe_convert_string_to_object(arr)
        tm.assert_numpy_array_equal(result, np.array(['x', 2], dtype=object))
        self.assertTrue(result.dtype == object)

def test_possibly_convert_objects_copy():
    values = np.array([1, 2])

    out = convert._possibly_convert_objects(values, copy=False)
    assert_true(values is out)

    out = convert._possibly_convert_objects(values, copy=True)
    assert_true(values is not out)

    values = np.array(['apply','banana'])
    out = convert._possibly_convert_objects(values, copy=False)
    assert_true(values is out)

    out = convert._possibly_convert_objects(values, copy=True)
    assert_true(values is not out)

def test_dict_compat():
    data_datetime64 = {np.datetime64('1990-03-15'): 1,
                       np.datetime64('2015-03-15'): 2}
    data_unchanged = {1: 2, 3: 4, 5: 6}
    expected = {Timestamp('1990-3-15'): 1, Timestamp('2015-03-15'): 2}
    assert(com._dict_compat(data_datetime64) == expected)
    assert(com._dict_compat(expected) == expected)
    assert(com._dict_compat(data_unchanged) == data_unchanged)

if __name__ == '__main__':
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)


#------------------------------------------------------------------------------
'''Class-style Model

This fixture is used to test the importing of models, handled by the
`theories.handshake()` module.  As of 0.4.11, models can:

- be located in the `lamana.models` folder
- module and classes can have any pythonic name; hard-coding removed
- any sub-package can be accessed by the "model" keyword in `Case.apply()`
- search for the hook-containing class and it's hook method

This module is here to test these aspects as the module is imported.  The Wilson_LT
model was adapted.  No functions are expected in this module; there are tests
against this.

'''

import math
import collections as ct

import pandas as pd

from lamana.input_ import BaseDefaults
from lamana.theories import BaseModel
from lamana.lt_exceptions import IndeterminateError

# This class lacks a hook method; theories should skip it.
class DummyModel():
	pass

# The class containing the hook method can have any name.
class RandomName(BaseModel):
    '''A modified laminate theory for circular biaxial flexure disks,
    loaded with a flat piston punch on 3-ball support having two distinct
    materials (polymer and ceramic).'''

    '''Accept extra args and kwds here'''
    def __init__(self):
        self.Laminate = None
        self.FeatureInput = None
        self.LaminateModel = None

    # TODO: eventually abstract into BaseModel and deprecate direct coding
    # TODO: accept kwargs from Case -> handshake
    def _use_model_(self, Laminate, adjusted_z=False):
        '''Return updated DataFrame and FeatureInput Return None if exceptions raised.

        Parameters
        ----------
        df : DataFrame
            LaminateModel with IDs and Dimensional Variables.
        FeatureInut : dict
            Geometry, laminate parameters and more.  Updates Globals dict for
            parameters in the dashboard output.
        adjusted_z: bool; default=False
            If True, uses z(m)* values instead; different assumption for internal calc.

        Raises
        ------
        ZeroDivisionError
            If zero `r` or `a` in the log term are zero.
        ValueError
            If negative numbers are in the log term or the support radius exceeds
            the sample radius.

        Returns
        -------
        tuple
            The updated calculations and parameters stored in a tuple
            `(LaminateModel, FeatureInput)``.

        '''
        self.Laminate = Laminate
        df = Laminate.LFrame.copy()
        FeatureInput = Laminate.FeatureInput

        # Author-defined Exception Handling
        if (FeatureInput['Parameters']['r'] == 0):
            raise ZeroDivisionError('r=0 is invalid for the log term in the moment eqn.')
        elif (FeatureInput['Parameters']['a'] == 0):
            raise ZeroDivisionError('a=0 is invalid for the log term in the moment eqn.')
        elif (FeatureInput['Parameters']['r'] < 0) | (FeatureInput['Parameters']['a'] < 0):
            raise ValueError('Negative numbers are invalid for the log term '
                             'in moment eqn.')
        elif FeatureInput['Parameters']['a'] > FeatureInput['Parameters']['R']:
            raise ValueError('Support radius is larger than sample radius.')
        elif df['side'].str.contains('INDET').any():
            print('INDET value found.  Rolling back...')
            raise IndeterminateError('INDET value found. Unable to accurately calculate stress.')
            #raise AssertionError('Indeterminate value found.  Unable to accurately calculate stress.')

        # Calling functions to calculate Qs and Ds
        df.loc[:, 'Q_11'] = self.calc_stiffness(df, FeatureInput['Properties']).q_11
        df.loc[:, 'Q_12'] = self.calc_stiffness(df, FeatureInput['Properties']).q_12
        df.loc[:, 'D_11'] = self.calc_bending(df, adj_z=adjusted_z).d_11
        df.loc[:, 'D_12'] = self.calc_bending(df, adj_z=adjusted_z).d_12

        # Global Variable Update
        if (FeatureInput['Parameters']['p'] == 1) & (Laminate.nplies%2 == 0):
            D_11T = sum(df['D_11'])
            D_12T = sum(df['D_12'])
        else:
            D_11T = sum(df.loc[df['label'] == 'interface', 'D_11']) # total D11
            D_12T = sum(df.loc[df['label'] == 'interface', 'D_12'])
        #print(FeatureInput['Geometric']['p'])

        D_11p = (1./((D_11T**2 - D_12T**2)) * D_11T)         #
        D_12n = -(1./((D_11T**2 - D_12T**2))  *D_12T)        #
        v_eq = D_12T/D_11T                                   # equiv. Poisson's ratio
        M_r = self.calc_moment(df, FeatureInput['Parameters'], v_eq).m_r
        M_t = self.calc_moment(df, FeatureInput['Parameters'], v_eq).m_t
        K_r = (D_11p*M_r) + (D_12n*M_t)                    # curvatures
        K_t = (D_12n*M_r) + (D_11p*M_t)

        # Update FeatureInput
        global_params = {
            'D_11T': D_11T,
            'D_12T': D_12T,
            'D_11p': D_11p,
            'D_12n': D_12n,
            'v_eq ': v_eq,
            'M_r': M_r,
            'M_t': M_t,
            'K_r': K_r,
            'K_t:': K_t,
        }

        FeatureInput['Globals'] = global_params
        self.FeatureInput = FeatureInput                   # update with Globals
        #print(FeatureInput)

        # Calculate Strains and Stresses and Update DataFrame
        df.loc[:,'strain_r'] = K_r * df.loc[:, 'Z(m)']
        df.loc[:,'strain_t'] = K_t * df.loc[:, 'Z(m)']
        df.loc[:, 'stress_r (Pa/N)'] = (df.loc[:, 'strain_r'] * df.loc[:, 'Q_11']
                                ) + (df.loc[:, 'strain_t'] * df.loc[:, 'Q_12'])
        df.loc[:,'stress_t (Pa/N)'] = (df.loc[:, 'strain_t'] * df.loc[:, 'Q_11']
                             ) + (df.loc[:, 'strain_r'] * df.loc[:, 'Q_12'])
        df.loc[:,'stress_f (MPa/N)'] = df.loc[:, 'stress_t (Pa/N)']/1e6

        del df['Modulus']
        del df['Poissons']

        self.LaminateModel = df

        return (df, FeatureInput)

    #------------------------------------------------------------------------------
    '''Prefer staticmethods here.  Add formulas to doc strings.'''
    def calc_stiffness(self, df, mat_props):
        '''Return tuple of Series of (Q11, Q12) floats per lamina.'''
        # Iterate to Apply Modulus and Poisson's to correct Material
        # TODO: Prefer cleaner ways to parse materials from mat_props
        df_mat_props = pd.DataFrame(mat_props)             # df easier to munge
        df_mat_props.index.name = 'materials'
        ##for material in mat_props.index:
        for material in df_mat_props.index:
            mat_idx = df['matl'] == material
            df.loc[mat_idx, 'Modulus'] = df_mat_props.loc[material, 'Modulus']
            df.loc[mat_idx, 'Poissons'] = df_mat_props.loc[material, 'Poissons']
            E = df['Modulus']                              # series of moduli
            v = df['Poissons']
        stiffness = ct.namedtuple('stiffness', ['q_11', 'q_12'])
        q_11 = E / (1 - (v**2))
        q_12 = (v*E) / (1 - (v**2))
        return stiffness(q_11, q_12)

    def calc_bending(self, df, adj_z=False):
        '''Return tuple of Series of (D11, D12) floats.'''
        q_11 = df['Q_11']
        q_12 = df['Q_12']
        h = df['h(m)']
        # TODO: need to fix kwargs passing first; tabled since affects many modules.
        if not adj_z:
            z = df['z(m)']
        else:
            z = df['z(m)*']
        bending = ct.namedtuple('bending', ['d_11', 'd_12'])
        d_11 = ((q_11*(h**3)) / 12.) + (q_11*h*(z**2))
        d_12 = ((q_12*(h**3)) / 12.) + (q_12*h*(z**2))
        return bending(d_11, d_12)

    def calc_moment(self, df, load_params, v_eq):
        '''Return tuple of moments (radial and tangential); floats.
        See Timishenko-Woinowsky: Eq. 91; default'''
        P_a = load_params['P_a']
        a = load_params['a']
        r = load_params['r']
        moments = ct.namedtuple('moments', ['m_r', 'm_t'])
        m_r = ((P_a/(4*math.pi)) * ((1 + v_eq)*math.log10(a/r)))
        m_t = ((P_a/(4*math.pi)) * (((1 + v_eq)*math.log10(a/r)) + (1 - v_eq)))
        return moments(m_r, m_t)

class Defaults(BaseDefaults):
    '''Return parameters for building distributions cases.  Useful for consistent
    testing.

    Dimensional defaults are inherited from utils.BaseDefaults().
    Material-specific parameters are defined here by he user.

    - Default geometric parameters
    - Default material properties
    - Default FeatureInput

    Examples
    --------
    >>> dft = Defaults()
    >>> dft.load_params
    {'R' : 12e-3, 'a' : 7.5e-3, 'p' : 1, 'P_a' : 1, 'r' : 2e-4,}

    >>> dft.mat_props
    {'Modulus': {'HA': 5.2e10, 'PSu': 2.7e9},
    'Poissons': {'HA': 0.25, 'PSu': 0.33}}

    >>> dft.FeatureInput
     {'Geometry' : '400-[200]-800',
      'Geometric' : {'R' : 12e-3, 'a' : 7.5e-3, 'p' : 1, 'P_a' : 1, 'r' : 2e-4,},
      'Materials' : {'HA' : [5.2e10, 0.25], 'PSu' : [2.7e9, 0.33],},
      'Custom' : None,
      'Model' : Wilson_LT}

    Returns
    -------
    class
        Updated attributes inherited from  the `BaseDefaults` class.

    '''
    def __init__(self):
        BaseDefaults.__init__(self)
        '''DEV: Add defaults first.  Then adjust attributes.'''
        # DEFAULTS ------------------------------------------------------------
        # Build dicts of geometric and material parameters
        self.load_params = {
            'R': 12e-3,                                    # specimen radius
            'a': 7.5e-3,                                   # support ring radius
            'p': 5,                                        # points/layer
            'P_a': 1,                                      # applied load
            'r': 2e-4,                                     # radial distance from center loading
        }

        self.mat_props = {
            'Modulus': {'HA': 5.2e10, 'PSu': 2.7e9},
            'Poissons': {'HA': 0.25, 'PSu': 0.33}
        }

        # ATTRIBUTES ----------------------------------------------------------
        # FeatureInput
        self.FeatureInput = self.get_FeatureInput(
            self.Geo_objects['standard'][0],
            load_params=self.load_params,
            mat_props=self.mat_props,
            ##custom_matls=None,
            model='Wilson_LT',
            global_vars=None
        )


"""Model evaluation tools."""

import os
import sklearn
import itertools

import numpy as np
import pandas as pd
import sklearn.metrics as skmetrics

from matplotlib import pyplot as plt

from healthcareai.common.healthcareai_error import HealthcareAIError

DIAGONAL_LINE_COLOR = '#bbbbbb'
DIAGONAL_LINE_STYLE = 'dotted'

def compute_roc(y_test, probability_predictions):
    """
    Compute TPRs, FPRs, best cutoff, ROC auc, and raw thresholds.

    Args:
        y_test (list) : true label values corresponding to the predictions. Also length n.
        probability_predictions (list) : predictions coming from an ML algorithm of length n.

    Returns:
        dict:

    """
    _validate_predictions_and_labels_are_equal_length(probability_predictions, y_test)

    # Calculate ROC
    false_positive_rates, true_positive_rates, roc_thresholds = skmetrics.roc_curve(y_test, probability_predictions)
    roc_auc = skmetrics.roc_auc_score(y_test, probability_predictions)

    # get ROC ideal cutoffs (upper left, or 0,1)
    roc_distances = (false_positive_rates - 0) ** 2 + (true_positive_rates - 1) ** 2

    # To prevent the case where there are two points with the same minimum distance, return only the first
    # np.where returns a tuple (we want the first element in the first array)
    roc_index = np.where(roc_distances == np.min(roc_distances))[0][0]
    best_tpr = true_positive_rates[roc_index]
    best_fpr = false_positive_rates[roc_index]
    ideal_roc_cutoff = roc_thresholds[roc_index]

    return {'roc_auc': roc_auc,
            'best_roc_cutoff': ideal_roc_cutoff,
            'best_true_positive_rate': best_tpr,
            'best_false_positive_rate': best_fpr,
            'true_positive_rates': true_positive_rates,
            'false_positive_rates': false_positive_rates,
            'roc_thresholds': roc_thresholds}

def compute_pr(y_test, probability_predictions):
    """
    Compute Precision-Recall, thresholds and PR AUC.

    Args:
        y_test (list) : true label values corresponding to the predictions. Also length n.
        probability_predictions (list) : predictions coming from an ML algorithm of length n.

    Returns:
        dict:

    """
    _validate_predictions_and_labels_are_equal_length(probability_predictions, y_test)

    # Calculate PR
    precisions, recalls, pr_thresholds = skmetrics.precision_recall_curve(y_test, probability_predictions)
    pr_auc = skmetrics.average_precision_score(y_test, probability_predictions)

    # get ideal cutoffs for suggestions (upper right or 1,1)
    pr_distances = (precisions - 1) ** 2 + (recalls - 1) ** 2

    # To prevent the case where there are two points with the same minimum distance, return only the first
    # np.where returns a tuple (we want the first element in the first array)
    pr_index = np.where(pr_distances == np.min(pr_distances))[0][0]
    best_precision = precisions[pr_index]
    best_recall = recalls[pr_index]
    ideal_pr_cutoff = pr_thresholds[pr_index]

    return {'pr_auc': pr_auc,
            'best_pr_cutoff': ideal_pr_cutoff,
            'best_precision': best_precision,
            'best_recall': best_recall,
            'precisions': precisions,
            'recalls': recalls,
            'pr_thresholds': pr_thresholds}

def calculate_regression_metrics(trained_sklearn_estimator, x_test, y_test):
    """
    Given a trained estimator, calculate metrics.

    Args:
        trained_sklearn_estimator (sklearn.base.BaseEstimator): a scikit-learn estimator that has been `.fit()`
        y_test (numpy.ndarray): A 1d numpy array of the y_test set (predictions)
        x_test (numpy.ndarray): A 2d numpy array of the x_test set (features)

    Returns:
        dict: A dictionary of metrics objects
    """
    # Get predictions
    predictions = trained_sklearn_estimator.predict(x_test)

    # Calculate individual metrics
    mean_squared_error = skmetrics.mean_squared_error(y_test, predictions)
    mean_absolute_error = skmetrics.mean_absolute_error(y_test, predictions)

    result = {'mean_squared_error': mean_squared_error, 'mean_absolute_error': mean_absolute_error}

    return result

def calculate_binary_classification_metrics(trained_sklearn_estimator, x_test, y_test):
    """
    Given a trained estimator, calculate metrics.

    Args:
        trained_sklearn_estimator (sklearn.base.BaseEstimator): a scikit-learn estimator that has been `.fit()`
        x_test (numpy.ndarray): A 2d numpy array of the x_test set (features)
        y_test (numpy.ndarray): A 1d numpy array of the y_test set (predictions)

    Returns:
        dict: A dictionary of metrics objects
    """
    # Squeeze down y_test to 1D
    y_test = np.squeeze(y_test)

    _validate_predictions_and_labels_are_equal_length(x_test, y_test)

    # Get binary and probability classification predictions
    binary_predictions = np.squeeze(trained_sklearn_estimator.predict(x_test))
    probability_predictions = np.squeeze(trained_sklearn_estimator.predict_proba(x_test)[:, 1])

    # Calculate accuracy
    accuracy = skmetrics.accuracy_score(y_test, binary_predictions)
    roc = compute_roc(y_test, probability_predictions)
    pr = compute_pr(y_test, probability_predictions)

    # Unpack the roc and pr dictionaries so the metric lookup is easier for plot and ensemble methods
    return {'accuracy': accuracy, **roc, **pr}

def roc_plot_from_thresholds(roc_thresholds_by_model, save=False, debug=False):
    """
    From a given dictionary of thresholds by model, create a ROC curve for each model.

    Args:
        roc_thresholds_by_model (dict): A dictionary of ROC thresholds by model name.
        save (bool): False to display the image (default) or True to save it (but not display it)
        debug (bool): verbost output.
    """
    # TODO consolidate this and PR plotter into 1 function
    # TODO make the colors randomly generated from rgb values
    # Cycle through the colors list
    color_iterator = itertools.cycle(['b', 'g', 'r', 'c', 'm', 'y', 'k'])
    # Initialize plot
    plt.figure()
    plt.xlabel('False Positive Rate (FPR)')
    plt.ylabel('True Positive Rate (TRP)')
    plt.title('Receiver Operating Characteristic (ROC)')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.plot([0, 1], [0, 1], linestyle=DIAGONAL_LINE_STYLE, color=DIAGONAL_LINE_COLOR)

    # Calculate and plot for each model
    for color, (model_name, metrics) in zip(color_iterator, roc_thresholds_by_model.items()):
        # Extract model name and metrics from dictionary
        roc_auc = metrics['roc_auc']
        tpr = metrics['true_positive_rates']
        fpr = metrics['false_positive_rates']
        best_true_positive_rate = metrics['best_true_positive_rate']
        best_false_positive_rate = metrics['best_false_positive_rate']

        if debug:
            print('{} model:'.format(model_name))
            print(pd.DataFrame({'FPR': fpr, 'TPR': tpr}))

        # plot the line
        label = '{} (ROC AUC = {})'.format(model_name, round(roc_auc, 2))
        plt.plot(fpr, tpr, color=color, label=label)
        plt.plot([best_false_positive_rate], [best_true_positive_rate], marker='*', markersize=10, color=color)

    plt.legend(loc="lower right")

    if save:
        plt.savefig('ROC.png')
        source_path = os.path.dirname(os.path.abspath(__file__))
        print('\nROC plot saved in: {}'.format(source_path))

    plt.show()

def pr_plot_from_thresholds(pr_thresholds_by_model, save=False, debug=False):
    """
    From a given dictionary of thresholds by model, create a PR curve for each model.

    Args:
        pr_thresholds_by_model (dict): A dictionary of PR thresholds by model name.
        save (bool): False to display the image (default) or True to save it (but not display it)
        debug (bool): verbost output.
    """
    # TODO consolidate this and PR plotter into 1 function
    # TODO make the colors randomly generated from rgb values
    # Cycle through the colors list
    color_iterator = itertools.cycle(['b', 'g', 'r', 'c', 'm', 'y', 'k'])
    # Initialize plot
    plt.figure()
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.title('Precision Recall (PR)')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.plot([0, 1], [1, 0], linestyle=DIAGONAL_LINE_STYLE, color=DIAGONAL_LINE_COLOR)

    # Calculate and plot for each model
    for color, (model_name, metrics) in zip(color_iterator, pr_thresholds_by_model.items()):
        # Extract model name and metrics from dictionary
        pr_auc = metrics['pr_auc']
        precision = metrics['precisions']
        recall = metrics['recalls']
        best_recall = metrics['best_recall']
        best_precision = metrics['best_precision']

        if debug:
            print('{} model:'.format(model_name))
            print(pd.DataFrame({'Recall': recall, 'Precision': precision}))

        # plot the line
        label = '{} (PR AUC = {})'.format(model_name, round(pr_auc, 2))
        plt.plot(recall, precision, color=color, label=label)
        plt.plot([best_recall], [best_precision], marker='*', markersize=10, color=color)

    plt.legend(loc="lower left")

    if save:
        plt.savefig('PR.png')
        source_path = os.path.dirname(os.path.abspath(__file__))
        print('\nPR plot saved in: {}'.format(source_path))

    plt.show()

def plot_random_forest_feature_importance(trained_random_forest, x_train, feature_names, feature_limit=15, save=False):
    """
    Given a random forest estimator, an x_train array, the feature names save or display a feature importance plot.

    Args:
        trained_random_forest (sklearn.ensemble.RandomForestClassifier or sklearn.ensemble.RandomForestRegressor):
        x_train (numpy.array): A 2D numpy array that was used for training
        feature_names (list): Column names in the x_train set
        feature_limit (int): Number of features to display on graph
        save (bool): True to save the plot, false to display it in a blocking thread
    """
    _validate_random_forest_estimator(trained_random_forest)

    # Sort the feature names and relative importances
    # TODO this portion could probably be extracted and tested, since the plot is difficult to test
    aggregate_features_importances = trained_random_forest.feature_importances_
    indices = np.argsort(aggregate_features_importances)[::-1]
    sorted_feature_names = [feature_names[i] for i in indices]

    # limit the plot to the top n features so it stays legible on models with lots of features
    subset_indices = indices[0:feature_limit]

    number_of_features = x_train.shape[1]

    # build a range using the lesser value
    max_features = min(number_of_features, feature_limit)
    x_axis_limit = range(max_features)

    # Get the standard deviations for error bars
    standard_deviations = _standard_deviations_of_importances(trained_random_forest)

    # Turn off matplotlib interactive mode
    plt.ioff()

    # Set up the plot and axes
    figure = plt.figure()
    plt.title('Top {} (of {}) Important Features'.format(max_features, number_of_features))
    plt.ylabel('Relative Importance')

    # Plot each feature
    plt.bar(
        # this should go as far as the model or limit whichever is less
        x_axis_limit,
        aggregate_features_importances[subset_indices],
        color="g",
        yerr=standard_deviations[subset_indices],
        align="center")

    plt.xticks(x_axis_limit, sorted_feature_names, rotation=90)
    # x axis scales by default
    # set y axis min to zero
    plt.ylim(ymin=0)
    # plt.tight_layout() # Do not use tight_layout until https://github.com/matplotlib/matplotlib/issues/5456 is fixed
    # Because long feature names cause this error

    # Save or display the plot
    if save:
        plt.savefig('FeatureImportances.png')
        source_path = os.path.dirname(os.path.abspath(__file__))
        print('\nFeature importance plot saved in: {}'.format(source_path))

        # Close the figure so it does not get displayed
        plt.close(figure)
    else:
        plt.show()

def _validate_random_forest_estimator(trained_random_forest):
    """
    Validate that an input is a random forest estimator and raise an error if it is not.

    Args:
        trained_random_forest: any input
    """
    is_rf_classifier = isinstance(trained_random_forest, sklearn.ensemble.RandomForestClassifier)
    is_rf_regressor = isinstance(trained_random_forest, sklearn.ensemble.RandomForestRegressor)

    if not (is_rf_classifier or is_rf_regressor):
        raise HealthcareAIError('Feature plotting only works with a scikit learn Random Forest estimator.')

def _standard_deviations_of_importances(trained_random_forest):
    """
    Given a scikit-learn trained random forest estimator, return the standard deviations of all feature importances.

    Args:
        trained_random_forest (sklearn.ensemble.RandomForestClassifier or sklearn.ensemble.RandomForestRegressor): the
            trained estimator

    Returns:
        list: A numeric list
    """
    # Get the individual feature importances from each tree to find the standard deviation for plotting error bars
    individual_feature_importances = [tree.feature_importances_ for tree in trained_random_forest.estimators_]
    standard_deviations = np.std(individual_feature_importances, axis=0)

    return standard_deviations

def _validate_predictions_and_labels_are_equal_length(predictions, true_values):
    if len(predictions) == len(true_values):
        return True
    else:
        raise HealthcareAIError('The number of predictions is not equal to the number of true_values.')

if __name__ == '__main__':
    pass


import unittest
import copy
import sys

from nose.tools import assert_true
import numpy as np
from numpy.testing import (assert_array_equal, assert_array_almost_equal,
                           assert_raises)
from scipy import stats
from sklearn import mixture
from sklearn.datasets.samples_generator import make_spd_matrix
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raise_message
from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.externals.six.moves import cStringIO as StringIO

rng = np.random.RandomState(0)

def test_sample_gaussian():
    # Test sample generation from mixture.sample_gaussian where covariance
    # is diagonal, spherical and full

    n_features, n_samples = 2, 300
    axis = 1
    mu = rng.randint(10) * rng.rand(n_features)
    cv = (rng.rand(n_features) + 1.0) ** 2

    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='diag', n_samples=n_samples)

    assert_true(np.allclose(samples.mean(axis), mu, atol=1.3))
    assert_true(np.allclose(samples.var(axis), cv, atol=1.5))

    # the same for spherical covariances
    cv = (rng.rand() + 1.0) ** 2
    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='spherical', n_samples=n_samples)

    assert_true(np.allclose(samples.mean(axis), mu, atol=1.5))
    assert_true(np.allclose(
        samples.var(axis), np.repeat(cv, n_features), atol=1.5))

    # and for full covariances
    A = rng.randn(n_features, n_features)
    cv = np.dot(A.T, A) + np.eye(n_features)
    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='full', n_samples=n_samples)
    assert_true(np.allclose(samples.mean(axis), mu, atol=1.3))
    assert_true(np.allclose(np.cov(samples), cv, atol=2.5))

    # Numerical stability check: in SciPy 0.12.0 at least, eigh may return
    # tiny negative values in its second return value.
    from sklearn.mixture import sample_gaussian
    x = sample_gaussian([0, 0], [[4, 3], [1, .1]],
                        covariance_type='full', random_state=42)
    print(x)
    assert_true(np.isfinite(x).all())

def _naive_lmvnpdf_diag(X, mu, cv):
    # slow and naive implementation of lmvnpdf
    ref = np.empty((len(X), len(mu)))
    stds = np.sqrt(cv)
    for i, (m, std) in enumerate(zip(mu, stds)):
        ref[:, i] = np.log(stats.norm.pdf(X, m, std)).sum(axis=1)
    return ref

def test_lmvnpdf_diag():
    # test a slow and naive implementation of lmvnpdf and
    # compare it to the vectorized version (mixture.lmvnpdf) to test
    # for correctness
    n_features, n_components, n_samples = 2, 3, 10
    mu = rng.randint(10) * rng.rand(n_components, n_features)
    cv = (rng.rand(n_components, n_features) + 1.0) ** 2
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    ref = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = mixture.log_multivariate_normal_density(X, mu, cv, 'diag')
    assert_array_almost_equal(lpr, ref)

def test_lmvnpdf_spherical():
    n_features, n_components, n_samples = 2, 3, 10

    mu = rng.randint(10) * rng.rand(n_components, n_features)
    spherecv = rng.rand(n_components, 1) ** 2 + 1
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    cv = np.tile(spherecv, (n_features, 1))
    reference = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = mixture.log_multivariate_normal_density(X, mu, spherecv,
                                                  'spherical')
    assert_array_almost_equal(lpr, reference)

def test_lmvnpdf_full():
    n_features, n_components, n_samples = 2, 3, 10

    mu = rng.randint(10) * rng.rand(n_components, n_features)
    cv = (rng.rand(n_components, n_features) + 1.0) ** 2
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    fullcv = np.array([np.diag(x) for x in cv])

    reference = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = mixture.log_multivariate_normal_density(X, mu, fullcv, 'full')
    assert_array_almost_equal(lpr, reference)

def test_lvmpdf_full_cv_non_positive_definite():
    n_features, n_samples = 2, 10
    rng = np.random.RandomState(0)
    X = rng.randint(10) * rng.rand(n_samples, n_features)
    mu = np.mean(X, 0)
    cv = np.array([[[-1, 0], [0, 1]]])
    expected_message = "'covars' must be symmetric, positive-definite"
    assert_raise_message(ValueError, expected_message,
                         mixture.log_multivariate_normal_density,
                         X, mu, cv, 'full')

def test_GMM_attributes():
    n_components, n_features = 10, 4
    covariance_type = 'diag'
    g = mixture.GMM(n_components, covariance_type, random_state=rng)
    weights = rng.rand(n_components)
    weights = weights / weights.sum()
    means = rng.randint(-20, 20, (n_components, n_features))

    assert_true(g.n_components == n_components)
    assert_true(g.covariance_type == covariance_type)

    g.weights_ = weights
    assert_array_almost_equal(g.weights_, weights)
    g.means_ = means
    assert_array_almost_equal(g.means_, means)

    covars = (0.1 + 2 * rng.rand(n_components, n_features)) ** 2
    g.covars_ = covars
    assert_array_almost_equal(g.covars_, covars)
    assert_raises(ValueError, g._set_covars, [])
    assert_raises(ValueError, g._set_covars,
                  np.zeros((n_components - 2, n_features)))

    assert_raises(ValueError, mixture.GMM, n_components=20,
                  covariance_type='badcovariance_type')

class GMMTester():
    do_test_eval = True

    def _setUp(self):
        self.n_components = 10
        self.n_features = 4
        self.weights = rng.rand(self.n_components)
        self.weights = self.weights / self.weights.sum()
        self.means = rng.randint(-20, 20, (self.n_components, self.n_features))
        self.threshold = -0.5
        self.I = np.eye(self.n_features)
        self.covars = {
            'spherical': (0.1 + 2 * rng.rand(self.n_components,
                                             self.n_features)) ** 2,
            'tied': (make_spd_matrix(self.n_features, random_state=0)
                     + 5 * self.I),
            'diag': (0.1 + 2 * rng.rand(self.n_components,
                                        self.n_features)) ** 2,
            'full': np.array([make_spd_matrix(self.n_features, random_state=0)
                              + 5 * self.I for x in range(self.n_components)])}

    def test_eval(self):
        if not self.do_test_eval:
            return  # DPGMM does not support setting the means and
        # covariances before fitting There is no way of fixing this
        # due to the variational parameters being more expressive than
        # covariance matrices
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type, random_state=rng)
        # Make sure the means are far apart so responsibilities.argmax()
        # picks the actual component used to generate the observations.
        g.means_ = 20 * self.means
        g.covars_ = self.covars[self.covariance_type]
        g.weights_ = self.weights

        gaussidx = np.repeat(np.arange(self.n_components), 5)
        n_samples = len(gaussidx)
        X = rng.randn(n_samples, self.n_features) + g.means_[gaussidx]

        ll, responsibilities = g.score_samples(X)

        self.assertEqual(len(ll), n_samples)
        self.assertEqual(responsibilities.shape,
                         (n_samples, self.n_components))
        assert_array_almost_equal(responsibilities.sum(axis=1),
                                  np.ones(n_samples))
        assert_array_equal(responsibilities.argmax(axis=1), gaussidx)

    def test_sample(self, n=100):
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type, random_state=rng)
        # Make sure the means are far apart so responsibilities.argmax()
        # picks the actual component used to generate the observations.
        g.means_ = 20 * self.means
        g.covars_ = np.maximum(self.covars[self.covariance_type], 0.1)
        g.weights_ = self.weights

        samples = g.sample(n)
        self.assertEqual(samples.shape, (n, self.n_features))

    def test_train(self, params='wmc'):
        g = mixture.GMM(n_components=self.n_components,
                        covariance_type=self.covariance_type)
        g.weights_ = self.weights
        g.means_ = self.means
        g.covars_ = 20 * self.covars[self.covariance_type]

        # Create a training set by sampling from the predefined distribution.
        X = g.sample(n_samples=100)
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-1,
                       n_iter=1, init_params=params)
        g.fit(X)

        # Do one training iteration at a time so we can keep track of
        # the log likelihood to make sure that it increases after each
        # iteration.
        trainll = []
        for _ in range(5):
            g.params = params
            g.init_params = ''
            g.fit(X)
            trainll.append(self.score(g, X))
        g.n_iter = 10
        g.init_params = ''
        g.params = params
        g.fit(X)  # finish fitting

        # Note that the log likelihood will sometimes decrease by a
        # very small amount after it has more or less converged due to
        # the addition of min_covar to the covariance (to prevent
        # underflow).  This is why the threshold is set to -0.5
        # instead of 0.
        delta_min = np.diff(trainll).min()
        self.assertTrue(
            delta_min > self.threshold,
            "The min nll increase is %f which is lower than the admissible"
            " threshold of %f, for model %s. The likelihoods are %s."
            % (delta_min, self.threshold, self.covariance_type, trainll))

    def test_train_degenerate(self, params='wmc'):
        # Train on degenerate data with 0 in some dimensions
        # Create a training set by sampling from the predefined distribution.
        X = rng.randn(100, self.n_features)
        X.T[1:] = 0
        g = self.model(n_components=2, covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-3, n_iter=5,
                       init_params=params)
        g.fit(X)
        trainll = g.score(X)
        self.assertTrue(np.sum(np.abs(trainll / 100 / X.shape[1])) < 5)

    def test_train_1d(self, params='wmc'):
        # Train on 1-D data
        # Create a training set by sampling from the predefined distribution.
        X = rng.randn(100, 1)
        # X.T[1:] = 0
        g = self.model(n_components=2, covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-7, n_iter=5,
                       init_params=params)
        g.fit(X)
        trainll = g.score(X)
        if isinstance(g, mixture.DPGMM):
            self.assertTrue(np.sum(np.abs(trainll / 100)) < 5)
        else:
            self.assertTrue(np.sum(np.abs(trainll / 100)) < 2)

    def score(self, g, X):
        return g.score(X).sum()

class TestGMMWithSphericalCovars(unittest.TestCase, GMMTester):
    covariance_type = 'spherical'
    model = mixture.GMM
    setUp = GMMTester._setUp

class TestGMMWithDiagonalCovars(unittest.TestCase, GMMTester):
    covariance_type = 'diag'
    model = mixture.GMM
    setUp = GMMTester._setUp

class TestGMMWithTiedCovars(unittest.TestCase, GMMTester):
    covariance_type = 'tied'
    model = mixture.GMM
    setUp = GMMTester._setUp

class TestGMMWithFullCovars(unittest.TestCase, GMMTester):
    covariance_type = 'full'
    model = mixture.GMM
    setUp = GMMTester._setUp

def test_multiple_init():
    # Test that multiple inits does not much worse than a single one
    X = rng.randn(30, 5)
    X[:10] += 2
    g = mixture.GMM(n_components=2, covariance_type='spherical',
                    random_state=rng, min_covar=1e-7, n_iter=5)
    train1 = g.fit(X).score(X).sum()
    g.n_init = 5
    train2 = g.fit(X).score(X).sum()
    assert_true(train2 >= train1 - 1.e-2)

def test_n_parameters():
    # Test that the right number of parameters is estimated
    n_samples, n_dim, n_components = 7, 5, 2
    X = rng.randn(n_samples, n_dim)
    n_params = {'spherical': 13, 'diag': 21, 'tied': 26, 'full': 41}
    for cv_type in ['full', 'tied', 'diag', 'spherical']:
        g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                        random_state=rng, min_covar=1e-7, n_iter=1)
        g.fit(X)
        assert_true(g._n_parameters() == n_params[cv_type])

def test_1d_1component():
    # Test all of the covariance_types return the same BIC score for
    # 1-dimensional, 1 component fits.
    n_samples, n_dim, n_components = 100, 1, 1
    X = rng.randn(n_samples, n_dim)
    g_full = mixture.GMM(n_components=n_components, covariance_type='full',
                         random_state=rng, min_covar=1e-7, n_iter=1)
    g_full.fit(X)
    g_full_bic = g_full.bic(X)
    for cv_type in ['tied', 'diag', 'spherical']:
        g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                        random_state=rng, min_covar=1e-7, n_iter=1)
        g.fit(X)
        assert_array_almost_equal(g.bic(X), g_full_bic)

def assert_fit_predict_correct(model, X):
    model2 = copy.deepcopy(model)

    predictions_1 = model.fit(X).predict(X)
    predictions_2 = model2.fit_predict(X)

    assert adjusted_rand_score(predictions_1, predictions_2) == 1.0

def test_fit_predict():
    """
    test that gmm.fit_predict is equivalent to gmm.fit + gmm.predict
    """
    lrng = np.random.RandomState(101)

    n_samples, n_dim, n_comps = 100, 2, 2
    mu = np.array([[8, 8]])
    component_0 = lrng.randn(n_samples, n_dim)
    component_1 = lrng.randn(n_samples, n_dim) + mu
    X = np.vstack((component_0, component_1))

    for m_constructor in (mixture.GMM, mixture.VBGMM, mixture.DPGMM):
        model = m_constructor(n_components=n_comps, covariance_type='full',
                              min_covar=1e-7, n_iter=5,
                              random_state=np.random.RandomState(0))
        assert_fit_predict_correct(model, X)

    model = mixture.GMM(n_components=n_comps, n_iter=0)
    z = model.fit_predict(X)
    assert np.all(z == 0), "Quick Initialization Failed!"

def test_aic():
    # Test the aic and bic criteria
    n_samples, n_dim, n_components = 50, 3, 2
    X = rng.randn(n_samples, n_dim)
    SGH = 0.5 * (X.var() + np.log(2 * np.pi))  # standard gaussian entropy

    for cv_type in ['full', 'tied', 'diag', 'spherical']:
        g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                        random_state=rng, min_covar=1e-7)
        g.fit(X)
        aic = 2 * n_samples * SGH * n_dim + 2 * g._n_parameters()
        bic = (2 * n_samples * SGH * n_dim +
               np.log(n_samples) * g._n_parameters())
        bound = n_dim * 3. / np.sqrt(n_samples)
        assert_true(np.abs(g.aic(X) - aic) / n_samples < bound)
        assert_true(np.abs(g.bic(X) - bic) / n_samples < bound)

def check_positive_definite_covars(covariance_type):
    r"""Test that covariance matrices do not become non positive definite

    Due to the accumulation of round-off errors, the computation of the
    covariance  matrices during the learning phase could lead to non-positive
    definite covariance matrices. Namely the use of the formula:

    .. math:: C = (\sum_i w_i  x_i x_i^T) - \mu \mu^T

    instead of:

    .. math:: C = \sum_i w_i (x_i - \mu)(x_i - \mu)^T

    while mathematically equivalent, was observed a ``LinAlgError`` exception,
    when computing a ``GMM`` with full covariance matrices and fixed mean.

    This function ensures that some later optimization will not introduce the
    problem again.
    """
    rng = np.random.RandomState(1)
    # we build a dataset with 2 2d component. The components are unbalanced
    # (respective weights 0.9 and 0.1)
    X = rng.randn(100, 2)
    X[-10:] += (3, 3)  # Shift the 10 last points

    gmm = mixture.GMM(2, params="wc", covariance_type=covariance_type,
                      min_covar=1e-3)

    # This is a non-regression test for issue #2640. The following call used
    # to trigger:
    # numpy.linalg.linalg.LinAlgError: 2-th leading minor not positive definite
    gmm.fit(X)

    if covariance_type == "diag" or covariance_type == "spherical":
        assert_greater(gmm.covars_.min(), 0)
    else:
        if covariance_type == "tied":
            covs = [gmm.covars_]
        else:
            covs = gmm.covars_

        for c in covs:
            assert_greater(np.linalg.det(c), 0)

def test_positive_definite_covars():
    # Check positive definiteness for all covariance types
    for covariance_type in ["full", "tied", "diag", "spherical"]:
        yield check_positive_definite_covars, covariance_type

def test_verbose_first_level():
    # Create sample data
    X = rng.randn(30, 5)
    X[:10] += 2
    g = mixture.GMM(n_components=2, n_init=2, verbose=1)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        g.fit(X)
    finally:
        sys.stdout = old_stdout

def test_verbose_second_level():
    # Create sample data
    X = rng.randn(30, 5)
    X[:10] += 2
    g = mixture.GMM(n_components=2, n_init=2, verbose=2)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        g.fit(X)
    finally:
        sys.stdout = old_stdout


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Ordinary Least Squares and Ridge Regression Variance
=========================================================
Due to the few points in each dimension and the straight
line that linear regression uses to follow these points
as well as it can, noise on the observations will cause
great variance as shown in the first plot. Every line's slope
can vary quite a bit for each prediction due to the noise
induced in the observations.

Ridge regression is basically minimizing a penalised version
of the least-squared function. The penalising `shrinks` the
value of the regression coefficients.
Despite the few data points in each dimension, the slope
of the prediction is much more stable and the variance
in the line itself is greatly reduced, in comparison to that
of the standard linear regression
"""
print(__doc__)

# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model

X_train = np.c_[.5, 1].T
y_train = [.5, 1]
X_test = np.c_[0, 2].T

np.random.seed(0)

classifiers = dict(ols=linear_model.LinearRegression(),
                   ridge=linear_model.Ridge(alpha=.1))

fignum = 1
for name, clf in classifiers.items():
    fig = plt.figure(fignum, figsize=(4, 3))
    plt.clf()
    plt.title(name)
    ax = plt.axes([.12, .12, .8, .8])

    for _ in range(6):
        this_X = .1 * np.random.normal(size=(2, 1)) + X_train
        clf.fit(this_X, y_train)

        ax.plot(X_test, clf.predict(X_test), color='.5')
        ax.scatter(this_X, y_train, s=3, c='.5', marker='o', zorder=10)

    clf.fit(X_train, y_train)
    ax.plot(X_test, clf.predict(X_test), linewidth=2, color='blue')
    ax.scatter(X_train, y_train, s=30, c='r', marker='+', zorder=10)

    ax.set_xticks(())
    ax.set_yticks(())
    ax.set_ylim((0, 1.6))
    ax.set_xlabel('X')
    ax.set_ylabel('y')
    ax.set_xlim(0, 2)
    fignum += 1

plt.show()


"""
=======================================================
Permutation F-test on sensor data with 1D cluster level
=======================================================

One tests if the evoked response is significantly different
between conditions. Multiple comparison problem is addressed
with cluster level permutation test.

"""
# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#
# License: BSD (3-clause)

import matplotlib.pyplot as plt

import mne
from mne import io
from mne.stats import permutation_cluster_test
from mne.datasets import sample

print(__doc__)

###############################################################################
# Set parameters
data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'
event_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw-eve.fif'
tmin = -0.2
tmax = 0.5

#   Setup for reading the raw data
raw = io.read_raw_fif(raw_fname)
events = mne.read_events(event_fname)

channel = 'MEG 1332'  # include only this channel in analysis
include = [channel]

###############################################################################
# Read epochs for the channel of interest
picks = mne.pick_types(raw.info, meg=False, eog=True, include=include,
                       exclude='bads')
event_id = 1
reject = dict(grad=4000e-13, eog=150e-6)
epochs1 = mne.Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                     baseline=(None, 0), reject=reject)
condition1 = epochs1.get_data()  # as 3D matrix

event_id = 2
epochs2 = mne.Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                     baseline=(None, 0), reject=reject)
condition2 = epochs2.get_data()  # as 3D matrix

condition1 = condition1[:, 0, :]  # take only one channel to get a 2D array
condition2 = condition2[:, 0, :]  # take only one channel to get a 2D array

###############################################################################
# Compute statistic
threshold = 6.0
T_obs, clusters, cluster_p_values, H0 = \
    permutation_cluster_test([condition1, condition2], n_permutations=1000,
                             threshold=threshold, tail=1, n_jobs=1,
                             out_type='mask')

###############################################################################
# Plot
times = epochs1.times
plt.close('all')
plt.subplot(211)
plt.title('Channel : ' + channel)
plt.plot(times, condition1.mean(axis=0) - condition2.mean(axis=0),
         label="ERF Contrast (Event 1 - Event 2)")
plt.ylabel("MEG (T / m)")
plt.legend()
plt.subplot(212)
for i_c, c in enumerate(clusters):
    c = c[0]
    if cluster_p_values[i_c] <= 0.05:
        h = plt.axvspan(times[c.start], times[c.stop - 1],
                        color='r', alpha=0.3)
    else:
        plt.axvspan(times[c.start], times[c.stop - 1], color=(0.3, 0.3, 0.3),
                    alpha=0.3)
hf = plt.plot(times, T_obs, 'g')
plt.legend((h, ), ('cluster p-value < 0.05', ))
plt.xlabel("time (ms)")
plt.ylabel("f-values")
plt.show()