from functools import partial
from datetime import datetime, time
from collections import MutableMapping

import numpy as np

from pandas._libs import tslib, tslibs
from pandas._libs.tslibs.strptime import array_strptime
from pandas._libs.tslibs import parsing, conversion, Timestamp
from pandas._libs.tslibs.parsing import (  # noqa
    parse_time_string,
    DateParseError,
    _format_is_iso,
    _guess_datetime_format)

from pandas.core.dtypes.common import (
    ensure_object,
    is_datetime64_ns_dtype,
    is_datetime64_dtype,
    is_datetime64tz_dtype,
    is_integer_dtype,
    is_integer,
    is_float,
    is_list_like,
    is_scalar,
    is_numeric_dtype,
    is_object_dtype)
from pandas.core.dtypes.generic import (
    ABCIndexClass, ABCSeries,
    ABCDataFrame)
from pandas.core.dtypes.missing import notna
from pandas.core import algorithms
from pandas.compat import zip

def _guess_datetime_format_for_array(arr, **kwargs):
    # Try to guess the format based on the first non-NaN element
    non_nan_elements = notna(arr).nonzero()[0]
    if len(non_nan_elements):
        return _guess_datetime_format(arr[non_nan_elements[0]], **kwargs)

def _maybe_cache(arg, format, cache, convert_listlike):
    """
    Create a cache of unique dates from an array of dates

    Parameters
    ----------
    arg : integer, float, string, datetime, list, tuple, 1-d array, Series
    format : string
        Strftime format to parse time
    cache : boolean
        True attempts to create a cache of converted values
    convert_listlike : function
        Conversion function to apply on dates

    Returns
    -------
    cache_array : Series
        Cache of converted, unique dates. Can be empty
    """
    from pandas import Series
    cache_array = Series()
    if cache:
        # Perform a quicker unique check
        from pandas import Index
        if not Index(arg).is_unique:
            unique_dates = algorithms.unique(arg)
            cache_dates = convert_listlike(unique_dates, True, format)
            cache_array = Series(cache_dates, index=unique_dates)
    return cache_array

def _convert_and_box_cache(arg, cache_array, box, errors, name=None):
    """
    Convert array of dates with a cache and box the result

    Parameters
    ----------
    arg : integer, float, string, datetime, list, tuple, 1-d array, Series
    cache_array : Series
        Cache of converted, unique dates
    box : boolean
        True boxes result as an Index-like, False returns an ndarray
    errors : string
        'ignore' plus box=True will convert result to Index
    name : string, default None
        Name for a DatetimeIndex

    Returns
    -------
    result : datetime of converted dates
        Returns:

        - Index-like if box=True
        - ndarray if box=False
    """
    from pandas import Series, DatetimeIndex, Index
    result = Series(arg).map(cache_array)
    if box:
        if errors == 'ignore':
            return Index(result, name=name)
        else:
            return DatetimeIndex(result, name=name)
    return result.values

def _return_parsed_timezone_results(result, timezones, box, tz, name):
    """
    Return results from array_strptime if a %z or %Z directive was passed.

    Parameters
    ----------
    result : ndarray
        int64 date representations of the dates
    timezones : ndarray
        pytz timezone objects
    box : boolean
        True boxes result as an Index-like, False returns an ndarray
    tz : object
        None or pytz timezone object
    name : string, default None
        Name for a DatetimeIndex

    Returns
    -------
    tz_result : ndarray of parsed dates with timezone
        Returns:

        - Index-like if box=True
        - ndarray of Timestamps if box=False

    """
    if tz is not None:
        raise ValueError("Cannot pass a tz argument when "
                         "parsing strings with timezone "
                         "information.")
    tz_results = np.array([Timestamp(res).tz_localize(zone) for res, zone
                           in zip(result, timezones)])
    if box:
        from pandas import Index
        return Index(tz_results, name=name)
    return tz_results

def _convert_listlike_datetimes(arg, box, format, name=None, tz=None,
                                unit=None, errors=None,
                                infer_datetime_format=None, dayfirst=None,
                                yearfirst=None, exact=None):
    """
    Helper function for to_datetime. Performs the conversions of 1D listlike
    of dates

    Parameters
    ----------
    arg : list, tuple, ndarray, Series, Index
        date to be parced
    box : boolean
        True boxes result as an Index-like, False returns an ndarray
    name : object
        None or string for the Index name
    tz : object
        None or 'utc'
    unit : string
        None or string of the frequency of the passed data
    errors : string
        error handing behaviors from to_datetime, 'raise', 'coerce', 'ignore'
    infer_datetime_format : boolean
        inferring format behavior from to_datetime
    dayfirst : boolean
        dayfirst parsing behavior from to_datetime
    yearfirst : boolean
        yearfirst parsing behavior from to_datetime
    exact : boolean
        exact format matching behavior from to_datetime

    Returns
    -------
    ndarray of parsed dates
        Returns:

        - Index-like if box=True
        - ndarray of Timestamps if box=False
    """
    from pandas import DatetimeIndex
    if isinstance(arg, (list, tuple)):
        arg = np.array(arg, dtype='O')

    # these are shortcutable
    if is_datetime64tz_dtype(arg):
        if not isinstance(arg, DatetimeIndex):
            return DatetimeIndex(arg, tz=tz, name=name)
        if tz == 'utc':
            arg = arg.tz_convert(None).tz_localize(tz)
        return arg

    elif is_datetime64_ns_dtype(arg):
        if box and not isinstance(arg, DatetimeIndex):
            try:
                return DatetimeIndex(arg, tz=tz, name=name)
            except ValueError:
                pass

        return arg

    elif unit is not None:
        if format is not None:
            raise ValueError("cannot specify both format and unit")
        arg = getattr(arg, 'values', arg)
        result = tslib.array_with_unit_to_datetime(arg, unit,
                                                   errors=errors)
        if box:
            if errors == 'ignore':
                from pandas import Index
                return Index(result, name=name)

            return DatetimeIndex(result, tz=tz, name=name)
        return result
    elif getattr(arg, 'ndim', 1) > 1:
        raise TypeError('arg must be a string, datetime, list, tuple, '
                        '1-d array, or Series')

    arg = ensure_object(arg)
    require_iso8601 = False

    if infer_datetime_format and format is None:
        format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)

    if format is not None:
        # There is a special fast-path for iso8601 formatted
        # datetime strings, so in those cases don't use the inferred
        # format because this path makes process slower in this
        # special case
        format_is_iso8601 = _format_is_iso(format)
        if format_is_iso8601:
            require_iso8601 = not infer_datetime_format
            format = None

    try:
        result = None

        if format is not None:
            # shortcut formatting here
            if format == '%Y%m%d':
                try:
                    result = _attempt_YYYYMMDD(arg, errors=errors)
                except (ValueError, TypeError, tslibs.OutOfBoundsDatetime):
                    raise ValueError("cannot convert the input to "
                                     "'%Y%m%d' date format")

            # fallback
            if result is None:
                try:
                    result, timezones = array_strptime(
                        arg, format, exact=exact, errors=errors)
                    if '%Z' in format or '%z' in format:
                        return _return_parsed_timezone_results(
                            result, timezones, box, tz, name)
                except tslibs.OutOfBoundsDatetime:
                    if errors == 'raise':
                        raise
                    result = arg
                except ValueError:
                    # if format was inferred, try falling back
                    # to array_to_datetime - terminate here
                    # for specified formats
                    if not infer_datetime_format:
                        if errors == 'raise':
                            raise
                        result = arg

        if result is None and (format is None or infer_datetime_format):
            result, tz_parsed = tslib.array_to_datetime(
                arg,
                errors=errors,
                utc=tz == 'utc',
                dayfirst=dayfirst,
                yearfirst=yearfirst,
                require_iso8601=require_iso8601
            )
            if tz_parsed is not None:
                if box:
                    # We can take a shortcut since the datetime64 numpy array
                    # is in UTC
                    return DatetimeIndex._simple_new(result, name=name,
                                                     tz=tz_parsed)
                else:
                    # Convert the datetime64 numpy array to an numpy array
                    # of datetime objects
                    result = [Timestamp(ts, tz=tz_parsed).to_pydatetime()
                              for ts in result]
                    return np.array(result, dtype=object)

        if box:
            # Ensure we return an Index in all cases where box=True
            if is_datetime64_dtype(result):
                return DatetimeIndex(result, tz=tz, name=name)
            elif is_object_dtype(result):
                # e.g. an Index of datetime objects
                from pandas import Index
                return Index(result, name=name)
        return result

    except ValueError as e:
        try:
            values, tz = conversion.datetime_to_datetime64(arg)
            return DatetimeIndex._simple_new(values, name=name, tz=tz)
        except (ValueError, TypeError):
            raise e

def _adjust_to_origin(arg, origin, unit):
    """
    Helper function for to_datetime.
    Adjust input argument to the specified origin

    Parameters
    ----------
    arg : list, tuple, ndarray, Series, Index
        date to be adjusted
    origin : 'julian' or Timestamp
        origin offset for the arg
    unit : string
        passed unit from to_datetime, must be 'D'

    Returns
    -------
    ndarray or scalar of adjusted date(s)
    """
    if origin == 'julian':
        original = arg
        j0 = Timestamp(0).to_julian_date()
        if unit != 'D':
            raise ValueError("unit must be 'D' for origin='julian'")
        try:
            arg = arg - j0
        except TypeError:
            raise ValueError("incompatible 'arg' type for given "
                             "'origin'='julian'")

        # premptively check this for a nice range
        j_max = Timestamp.max.to_julian_date() - j0
        j_min = Timestamp.min.to_julian_date() - j0
        if np.any(arg > j_max) or np.any(arg < j_min):
            raise tslibs.OutOfBoundsDatetime(
                "{original} is Out of Bounds for "
                "origin='julian'".format(original=original))
    else:
        # arg must be numeric
        if not ((is_scalar(arg) and (is_integer(arg) or is_float(arg))) or
                is_numeric_dtype(np.asarray(arg))):
            raise ValueError(
                "'{arg}' is not compatible with origin='{origin}'; "
                "it must be numeric with a unit specified ".format(
                    arg=arg,
                    origin=origin))

        # we are going to offset back to unix / epoch time
        try:
            offset = Timestamp(origin)
        except tslibs.OutOfBoundsDatetime:
            raise tslibs.OutOfBoundsDatetime(
                "origin {origin} is Out of Bounds".format(origin=origin))
        except ValueError:
            raise ValueError("origin {origin} cannot be converted "
                             "to a Timestamp".format(origin=origin))

        if offset.tz is not None:
            raise ValueError(
                "origin offset {} must be tz-naive".format(offset))
        offset -= Timestamp(0)

        # convert the offset to the unit of the arg
        # this should be lossless in terms of precision
        offset = offset // tslibs.Timedelta(1, unit=unit)

        # scalars & ndarray-like can handle the addition
        if is_list_like(arg) and not isinstance(
                arg, (ABCSeries, ABCIndexClass, np.ndarray)):
            arg = np.asarray(arg)
        arg = arg + offset
    return arg

def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
                utc=None, box=True, format=None, exact=True,
                unit=None, infer_datetime_format=False, origin='unix',
                cache=False):
    """
    Convert argument to datetime.

    Parameters
    ----------
    arg : integer, float, string, datetime, list, tuple, 1-d array, Series

        .. versionadded:: 0.18.1

           or DataFrame/dict-like

    errors : {'ignore', 'raise', 'coerce'}, default 'raise'

        - If 'raise', then invalid parsing will raise an exception
        - If 'coerce', then invalid parsing will be set as NaT
        - If 'ignore', then invalid parsing will return the input
    dayfirst : boolean, default False
        Specify a date parse order if `arg` is str or its list-likes.
        If True, parses dates with the day first, eg 10/11/12 is parsed as
        2012-11-10.
        Warning: dayfirst=True is not strict, but will prefer to parse
        with day first (this is a known bug, based on dateutil behavior).
    yearfirst : boolean, default False
        Specify a date parse order if `arg` is str or its list-likes.

        - If True parses dates with the year first, eg 10/11/12 is parsed as
          2010-11-12.
        - If both dayfirst and yearfirst are True, yearfirst is preceded (same
          as dateutil).

        Warning: yearfirst=True is not strict, but will prefer to parse
        with year first (this is a known bug, based on dateutil behavior).

        .. versionadded:: 0.16.1

    utc : boolean, default None
        Return UTC DatetimeIndex if True (converting any tz-aware
        datetime.datetime objects as well).
    box : boolean, default True

        - If True returns a DatetimeIndex or Index-like object
        - If False returns ndarray of values.
    format : string, default None
        strftime to parse time, eg "%d/%m/%Y", note that "%f" will parse
        all the way up to nanoseconds.
    exact : boolean, True by default

        - If True, require an exact format match.
        - If False, allow the format to match anywhere in the target string.

    unit : string, default 'ns'
        unit of the arg (D,s,ms,us,ns) denote the unit, which is an
        integer or float number. This will be based off the origin.
        Example, with unit='ms' and origin='unix' (the default), this
        would calculate the number of milliseconds to the unix epoch start.
    infer_datetime_format : boolean, default False
        If True and no `format` is given, attempt to infer the format of the
        datetime strings, and if it can be inferred, switch to a faster
        method of parsing them. In some cases this can increase the parsing
        speed by ~5-10x.
    origin : scalar, default is 'unix'
        Define the reference date. The numeric values would be parsed as number
        of units (defined by `unit`) since this reference date.

        - If 'unix' (or POSIX) time; origin is set to 1970-01-01.
        - If 'julian', unit must be 'D', and origin is set to beginning of
          Julian Calendar. Julian day number 0 is assigned to the day starting
          at noon on January 1, 4713 BC.
        - If Timestamp convertible, origin is set to Timestamp identified by
          origin.

        .. versionadded:: 0.20.0
    cache : boolean, default False
        If True, use a cache of unique, converted dates to apply the datetime
        conversion. May produce significant speed-up when parsing duplicate
        date strings, especially ones with timezone offsets.

        .. versionadded:: 0.23.0

    Returns
    -------
    ret : datetime if parsing succeeded.
        Return type depends on input:

        - list-like: DatetimeIndex
        - Series: Series of datetime64 dtype
        - scalar: Timestamp

        In case when it is not possible to return designated types (e.g. when
        any element of input is before Timestamp.min or after Timestamp.max)
        return will have datetime.datetime type (or corresponding
        array/Series).

    Examples
    --------
    Assembling a datetime from multiple columns of a DataFrame. The keys can be
    common abbreviations like ['year', 'month', 'day', 'minute', 'second',
    'ms', 'us', 'ns']) or plurals of the same

    >>> df = pd.DataFrame({'year': [2015, 2016],
                           'month': [2, 3],
                           'day': [4, 5]})
    >>> pd.to_datetime(df)
    0   2015-02-04
    1   2016-03-05
    dtype: datetime64[ns]

    If a date does not meet the `timestamp limitations
    <http://pandas.pydata.org/pandas-docs/stable/timeseries.html
    #timeseries-timestamp-limits>`_, passing errors='ignore'
    will return the original input instead of raising any exception.

    Passing errors='coerce' will force an out-of-bounds date to NaT,
    in addition to forcing non-dates (or non-parseable dates) to NaT.

    >>> pd.to_datetime('13000101', format='%Y%m%d', errors='ignore')
    datetime.datetime(1300, 1, 1, 0, 0)
    >>> pd.to_datetime('13000101', format='%Y%m%d', errors='coerce')
    NaT

    Passing infer_datetime_format=True can often-times speedup a parsing
    if its not an ISO8601 format exactly, but in a regular format.

    >>> s = pd.Series(['3/11/2000', '3/12/2000', '3/13/2000']*1000)

    >>> s.head()
    0    3/11/2000
    1    3/12/2000
    2    3/13/2000
    3    3/11/2000
    4    3/12/2000
    dtype: object

    >>> %timeit pd.to_datetime(s,infer_datetime_format=True)
    100 loops, best of 3: 10.4 ms per loop

    >>> %timeit pd.to_datetime(s,infer_datetime_format=False)
    1 loop, best of 3: 471 ms per loop

    Using a unix epoch time

    >>> pd.to_datetime(1490195805, unit='s')
    Timestamp('2017-03-22 15:16:45')
    >>> pd.to_datetime(1490195805433502912, unit='ns')
    Timestamp('2017-03-22 15:16:45.433502912')

    .. warning:: For float arg, precision rounding might happen. To prevent
        unexpected behavior use a fixed-width exact type.

    Using a non-unix epoch origin

    >>> pd.to_datetime([1, 2, 3], unit='D',
                       origin=pd.Timestamp('1960-01-01'))
    0    1960-01-02
    1    1960-01-03
    2    1960-01-04

    See also
    --------
    pandas.DataFrame.astype : Cast argument to a specified dtype.
    pandas.to_timedelta : Convert argument to timedelta.
    """
    if arg is None:
        return None

    if origin != 'unix':
        arg = _adjust_to_origin(arg, origin, unit)

    tz = 'utc' if utc else None
    convert_listlike = partial(_convert_listlike_datetimes, tz=tz, unit=unit,
                               dayfirst=dayfirst, yearfirst=yearfirst,
                               errors=errors, exact=exact,
                               infer_datetime_format=infer_datetime_format)

    if isinstance(arg, Timestamp):
        result = arg
    elif isinstance(arg, ABCSeries):
        cache_array = _maybe_cache(arg, format, cache, convert_listlike)
        if not cache_array.empty:
            result = arg.map(cache_array)
        else:
            from pandas import Series
            values = convert_listlike(arg._values, True, format)
            result = Series(values, index=arg.index, name=arg.name)
    elif isinstance(arg, (ABCDataFrame, MutableMapping)):
        result = _assemble_from_unit_mappings(arg, errors=errors)
    elif isinstance(arg, ABCIndexClass):
        cache_array = _maybe_cache(arg, format, cache, convert_listlike)
        if not cache_array.empty:
            result = _convert_and_box_cache(arg, cache_array, box, errors,
                                            name=arg.name)
        else:
            convert_listlike = partial(convert_listlike, name=arg.name)
            result = convert_listlike(arg, box, format)
    elif is_list_like(arg):
        cache_array = _maybe_cache(arg, format, cache, convert_listlike)
        if not cache_array.empty:
            result = _convert_and_box_cache(arg, cache_array, box, errors)
        else:
            result = convert_listlike(arg, box, format)
    else:
        result = convert_listlike(np.array([arg]), box, format)[0]

    return result

# mappings for assembling units
_unit_map = {'year': 'year',
             'years': 'year',
             'month': 'month',
             'months': 'month',
             'day': 'day',
             'days': 'day',
             'hour': 'h',
             'hours': 'h',
             'minute': 'm',
             'minutes': 'm',
             'second': 's',
             'seconds': 's',
             'ms': 'ms',
             'millisecond': 'ms',
             'milliseconds': 'ms',
             'us': 'us',
             'microsecond': 'us',
             'microseconds': 'us',
             'ns': 'ns',
             'nanosecond': 'ns',
             'nanoseconds': 'ns'
             }

def _assemble_from_unit_mappings(arg, errors):
    """
    assemble the unit specified fields from the arg (DataFrame)
    Return a Series for actual parsing

    Parameters
    ----------
    arg : DataFrame
    errors : {'ignore', 'raise', 'coerce'}, default 'raise'

        - If 'raise', then invalid parsing will raise an exception
        - If 'coerce', then invalid parsing will be set as NaT
        - If 'ignore', then invalid parsing will return the input

    Returns
    -------
    Series
    """
    from pandas import to_timedelta, to_numeric, DataFrame
    arg = DataFrame(arg)
    if not arg.columns.is_unique:
        raise ValueError("cannot assemble with duplicate keys")

    # replace passed unit with _unit_map
    def f(value):
        if value in _unit_map:
            return _unit_map[value]

        # m is case significant
        if value.lower() in _unit_map:
            return _unit_map[value.lower()]

        return value

    unit = {k: f(k) for k in arg.keys()}
    unit_rev = {v: k for k, v in unit.items()}

    # we require at least Ymd
    required = ['year', 'month', 'day']
    req = sorted(list(set(required) - set(unit_rev.keys())))
    if len(req):
        raise ValueError("to assemble mappings requires at least that "
                         "[year, month, day] be specified: [{required}] "
                         "is missing".format(required=','.join(req)))

    # keys we don't recognize
    excess = sorted(list(set(unit_rev.keys()) - set(_unit_map.values())))
    if len(excess):
        raise ValueError("extra keys have been passed "
                         "to the datetime assemblage: "
                         "[{excess}]".format(excess=','.join(excess)))

    def coerce(values):
        # we allow coercion to if errors allows
        values = to_numeric(values, errors=errors)

        # prevent overflow in case of int8 or int16
        if is_integer_dtype(values):
            values = values.astype('int64', copy=False)
        return values

    values = (coerce(arg[unit_rev['year']]) * 10000 +
              coerce(arg[unit_rev['month']]) * 100 +
              coerce(arg[unit_rev['day']]))
    try:
        values = to_datetime(values, format='%Y%m%d', errors=errors)
    except (TypeError, ValueError) as e:
        raise ValueError("cannot assemble the "
                         "datetimes: {error}".format(error=e))

    for u in ['h', 'm', 's', 'ms', 'us', 'ns']:
        value = unit_rev.get(u)
        if value is not None and value in arg:
            try:
                values += to_timedelta(coerce(arg[value]),
                                       unit=u,
                                       errors=errors)
            except (TypeError, ValueError) as e:
                raise ValueError("cannot assemble the datetimes [{value}]: "
                                 "{error}".format(value=value, error=e))

    return values

def _attempt_YYYYMMDD(arg, errors):
    """ try to parse the YYYYMMDD/%Y%m%d format, try to deal with NaT-like,
        arg is a passed in as an object dtype, but could really be ints/strings
        with nan-like/or floats (e.g. with nan)

    Parameters
    ----------
    arg : passed value
    errors : 'raise','ignore','coerce'
    """

    def calc(carg):
        # calculate the actual result
        carg = carg.astype(object)
        parsed = parsing.try_parse_year_month_day(carg / 10000,
                                                  carg / 100 % 100,
                                                  carg % 100)
        return tslib.array_to_datetime(parsed, errors=errors)[0]

    def calc_with_mask(carg, mask):
        result = np.empty(carg.shape, dtype='M8[ns]')
        iresult = result.view('i8')
        iresult[~mask] = tslibs.iNaT

        masked_result = calc(carg[mask].astype(np.float64).astype(np.int64))
        result[mask] = masked_result.astype('M8[ns]')
        return result

    # try intlike / strings that are ints
    try:
        return calc(arg.astype(np.int64))
    except ValueError:
        pass

    # a float with actual np.nan
    try:
        carg = arg.astype(np.float64)
        return calc_with_mask(carg, notna(carg))
    except ValueError:
        pass

    # string with NaN-like
    try:
        mask = ~algorithms.isin(arg, list(tslib.nat_strings))
        return calc_with_mask(arg, mask)
    except ValueError:
        pass

    return None

# Fixed time formats for time parsing
_time_formats = ["%H:%M", "%H%M", "%I:%M%p", "%I%M%p",
                 "%H:%M:%S", "%H%M%S", "%I:%M:%S%p", "%I%M%S%p"]

def _guess_time_format_for_array(arr):
    # Try to guess the format based on the first non-NaN element
    non_nan_elements = notna(arr).nonzero()[0]
    if len(non_nan_elements):
        element = arr[non_nan_elements[0]]
        for time_format in _time_formats:
            try:
                datetime.strptime(element, time_format)
                return time_format
            except ValueError:
                pass

    return None

def to_time(arg, format=None, infer_time_format=False, errors='raise'):
    """
    Parse time strings to time objects using fixed strptime formats ("%H:%M",
    "%H%M", "%I:%M%p", "%I%M%p", "%H:%M:%S", "%H%M%S", "%I:%M:%S%p",
    "%I%M%S%p")

    Use infer_time_format if all the strings are in the same format to speed
    up conversion.

    Parameters
    ----------
    arg : string in time format, datetime.time, list, tuple, 1-d array,  Series
    format : str, default None
        Format used to convert arg into a time object.  If None, fixed formats
        are used.
    infer_time_format: bool, default False
        Infer the time format based on the first non-NaN element.  If all
        strings are in the same format, this will speed up conversion.
    errors : {'ignore', 'raise', 'coerce'}, default 'raise'
        - If 'raise', then invalid parsing will raise an exception
        - If 'coerce', then invalid parsing will be set as None
        - If 'ignore', then invalid parsing will return the input

    Returns
    -------
    datetime.time
    """
    from pandas.core.series import Series

    def _convert_listlike(arg, format):

        if isinstance(arg, (list, tuple)):
            arg = np.array(arg, dtype='O')

        elif getattr(arg, 'ndim', 1) > 1:
            raise TypeError('arg must be a string, datetime, list, tuple, '
                            '1-d array, or Series')

        arg = ensure_object(arg)

        if infer_time_format and format is None:
            format = _guess_time_format_for_array(arg)

        times = []
        if format is not None:
            for element in arg:
                try:
                    times.append(datetime.strptime(element, format).time())
                except (ValueError, TypeError):
                    if errors == 'raise':
                        msg = ("Cannot convert {element} to a time with given "
                               "format {format}").format(element=element,
                                                         format=format)
                        raise ValueError(msg)
                    elif errors == 'ignore':
                        return arg
                    else:
                        times.append(None)
        else:
            formats = _time_formats[:]
            format_found = False
            for element in arg:
                time_object = None
                for time_format in formats:
                    try:
                        time_object = datetime.strptime(element,
                                                        time_format).time()
                        if not format_found:
                            # Put the found format in front
                            fmt = formats.pop(formats.index(time_format))
                            formats.insert(0, fmt)
                            format_found = True
                        break
                    except (ValueError, TypeError):
                        continue

                if time_object is not None:
                    times.append(time_object)
                elif errors == 'raise':
                    raise ValueError("Cannot convert arg {arg} to "
                                     "a time".format(arg=arg))
                elif errors == 'ignore':
                    return arg
                else:
                    times.append(None)

        return times

    if arg is None:
        return arg
    elif isinstance(arg, time):
        return arg
    elif isinstance(arg, Series):
        values = _convert_listlike(arg._values, format)
        return Series(values, index=arg.index, name=arg.name)
    elif isinstance(arg, ABCIndexClass):
        return _convert_listlike(arg, format)
    elif is_list_like(arg):
        return _convert_listlike(arg, format)

    return _convert_listlike(np.array([arg]), format)[0]


# Lab 7 Learning rate and Evaluation
import tensorflow as tf
import random
import matplotlib.pyplot as plt
tf.set_random_seed(777) # for reproducibility

from tensorflow.examples.tutorials.mnist import input_data
# Check out https://www.tensorflow.org/get_started/mnist/beginners for
# more information about the mnist dataset
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

nb_classes = 10

# MNIST data image of shape 28 * 28 = 784
X = tf.placeholder(tf.float32, [None, 784])
# 0 - 9 digits recognition = 10 classes
Y = tf.placeholder(tf.float32, [None, nb_classes])

W = tf.Variable(tf.random_normal([784, nb_classes]))
b = tf.Variable(tf.random_normal([nb_classes]))

# Hypothesis (using softmax)
hypothesis = tf.nn.softmax(tf.matmul(X, W) + b)

cost = tf.reduce_mean(-tf.reduce_sum(Y * tf.log(hypothesis), axis=1))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(cost)

# Test model
is_correct = tf.equal(tf.arg_max(hypothesis, 1), tf.arg_max(Y, 1))
# Calculate accuracy
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))

# parameters
training_epochs = 15
batch_size = 100

with tf.Session() as sess:
    # Initialize TensorFlow variables
    sess.run(tf.global_variables_initializer())
    # Training cycle
    for epoch in range(training_epochs):
        avg_cost = 0
        total_batch = int(mnist.train.num_examples / batch_size)

        for i in range(total_batch):
            batch_xs, batch_ys = mnist.train.next_batch(batch_size)
            c, _ = sess.run([cost, optimizer], feed_dict={
                X: batch_xs, Y: batch_ys})
            avg_cost += c / total_batch

print('Epoch:', '%04d' % (epoch + 1),
      'cost =', '{:.9f}'.format(avg_cost))

print("Learning finished")

# Test the model using test sets
print("Accuracy: ", accuracy.eval(session=sess, feed_dict={
    X: mnist.test.images, Y: mnist.test.labels}))

# Get one and predict
r = random.randint(0, mnist.test.num_examples - 1)
print("Label: ", sess.run(tf.argmax(mnist.test.labels[r:r + 1], 1)))
print("Prediction: ", sess.run(
    tf.argmax(hypothesis, 1), feed_dict={X: mnist.test.images[r:r + 1]}))

plt.imshow(
    mnist.test.images[r:r + 1].reshape(28, 28),
    cmap='Greys',
    interpolation='nearest')
plt.show()


"""
Helper for testing.
"""

import sys
import warnings
import os.path
import re
import subprocess
import threading

from sklearn.externals.joblib._compat import PY3_OR_LATER

def warnings_to_stdout():
    """ Redirect all warnings to stdout.
    """
    showwarning_orig = warnings.showwarning

    def showwarning(msg, cat, fname, lno, file=None, line=0):
        showwarning_orig(msg, cat, os.path.basename(fname), line, sys.stdout)

    warnings.showwarning = showwarning
    #warnings.simplefilter('always')

try:
    from nose.tools import assert_raises_regex
except ImportError:
    # For Python 2.7
    try:
        from nose.tools import assert_raises_regexp as assert_raises_regex
    except ImportError:
        # for Python 2.6
        def assert_raises_regex(expected_exception, expected_regexp,
                                callable_obj=None, *args, **kwargs):
            """Helper function to check for message patterns in exceptions"""

            not_raised = False
            try:
                callable_obj(*args, **kwargs)
                not_raised = True
            except Exception as e:
                error_message = str(e)
                if not re.compile(expected_regexp).search(error_message):
                    raise AssertionError("Error message should match pattern "
                                         "%r. %r does not." %
                                         (expected_regexp, error_message))
            if not_raised:
                raise AssertionError("Should have raised %r" %
                                     expected_exception(expected_regexp))

def check_subprocess_call(cmd, timeout=1, stdout_regex=None,
                          stderr_regex=None):
    """Runs a command in a subprocess with timeout in seconds.

    Also checks returncode is zero, stdout if stdout_regex is set, and
    stderr if stderr_regex is set.
    """
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)

    def kill_process():
        proc.kill()

    timer = threading.Timer(timeout, kill_process)
    try:
        timer.start()
        stdout, stderr = proc.communicate()

        if PY3_OR_LATER:
            stdout, stderr = stdout.decode(), stderr.decode()
        if proc.returncode != 0:
            message = (
                'Non-zero return code: {0}.\nStdout:\n{1}\n'
                'Stderr:\n{2}').format(
                    proc.returncode, stdout, stderr)
            raise ValueError(message)

        if (stdout_regex is not None and
                not re.search(stdout_regex, stdout)):
            raise ValueError(
                "Unexpected stdout: {0!r} does not match:\n{1!r}".format(
                    stdout_regex, stdout))
        if (stderr_regex is not None and
                not re.search(stderr_regex, stderr)):
            raise ValueError(
                "Unexpected stderr: {0!r} does not match:\n{1!r}".format(
                    stderr_regex, stderr))

    finally:
        timer.cancel()


"""
The :mod:`sklearn.feature_selection` module implements feature selection
algorithms. It currently includes univariate filter selection methods and the
recursive feature elimination algorithm.
"""

from .univariate_selection import chi2
from .univariate_selection import f_classif
from .univariate_selection import f_oneway
from .univariate_selection import f_regression
from .univariate_selection import SelectPercentile
from .univariate_selection import SelectKBest
from .univariate_selection import SelectFpr
from .univariate_selection import SelectFdr
from .univariate_selection import SelectFwe
from .univariate_selection import GenericUnivariateSelect

from .variance_threshold import VarianceThreshold

from .rfe import RFE
from .rfe import RFECV

__all__ = ['GenericUnivariateSelect',
           'RFE',
           'RFECV',
           'SelectFdr',
           'SelectFpr',
           'SelectFwe',
           'SelectKBest',
           'SelectPercentile',
           'VarianceThreshold',
           'chi2',
           'f_classif',
           'f_oneway',
           'f_regression']


#!/usr/bin/env python3
from __future__ import print_function
from __future__ import division
import argparse
import pickle
import time
import CCBuilder as ccb
import CCBuilder_c as ccb_c
import numpy as np
import scipy.special

def uniform_dist(x):
    """ Returns uniform distributions of given range """
    return lambda: np.random.uniform(x[0], x[1])

def weibull_dist(a, mu):
    """ Returns Weibull distributions for given shape parameter and average """
    return lambda: np.random.weibull(a) * mu / scipy.special.gamma(1/a + 1)

def parse_dist(arg):
    # Parses input string for given distribution.
    # Returns a distribution, and the average
    d, params = arg.split(':')
    params = [float(x) for x in params.split(',')]
    if d == 'U':
        return uniform_dist(params), np.mean(params)
    elif d == 'W':
        a, mu = params
        return weibull_dist(a, mu), mu

parser = argparse.ArgumentParser(description='''Generate a WC microstructure.
Grain shape/size supports 2 types of distributions:
Uniform: U:low,high
Weibull: U:a,mu     (a=k in some notation, mu=mean)
''',
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)

# parser.add_argument('-V', dest='verbose', action='store_true', help='Verbose mode.')

parser.add_argument('-f', dest='fname', metavar='basename', required=True, help='Output base filename.')

parser.add_argument('-L', dest='L', metavar='length', required=True, type=float, help='Cell length (volume is L^3)')

parser.add_argument('-m', dest='m', metavar='m', required=True, type=int,
                    help='Grid resolution. Total number of voxels are (m*L)^3')

parser.add_argument('--vol_frac_goal', dest="vol_frac_goal", metavar='v', required=True, type=float,
                    help='Goal for volume fraction WC (excluding overlap)')

parser.add_argument('-s', dest='seed', metavar='s', default=None, type=int,
                    help='Seed for RNG. Given identical parameters, ' +
                         'CCBuilder will generate identical output given a controlled seed.')

parser.add_argument('--stray_cleanup', action='store_true', help='Clean up stray voxels')

group = parser.add_argument_group('WC grain shape')
group.add_argument('-k', dest='k_dist', metavar='type,[params]', default='U:0.4,1.4',
                   help='k distribution')
group.add_argument('-r', dest='r_dist', metavar='type,[params]', default='U:0.1,0.4',
                   help='r distribution')
group.add_argument('-d', dest='d_dist', metavar='type,[params]', default='U:0.5,1.5',
                   help='d distribution')

group = parser.add_argument_group('Packing')
group.add_argument('--use_potential', action='store_true', help='Use repulsive potential.')
group.add_argument('--nr_tries', dest='nr_tries', metavar='n', default=2500, type=int,
                   help='Number of random translations.')
group.add_argument('--delta', dest='delta', metavar='d', type=float,
                   help='Maximum distance for randomized translations.')
group.add_argument('--m_coarse', dest="m_coarse", metavar='mc', default=10,
                   help='Grid resolution during packing.')

group = parser.add_argument_group('Potts simulation')
group.add_argument('--mc_steps', dest="mc_steps", metavar='steps', default=0.05, type=float,
                   help='Monte-Carlo steps (scales with (m*L)^4. Set to zero to turn off.')
group.add_argument('--tau', dest='tau', metavar='t', default=0.5, type=float,
                   help='Ficticious temperature in Potts model.')

options = parser.parse_args()

if options.seed is not None:
    np.random.seed(options.seed)

# Heuristic mapping from actual to goal volume fraction
# vol_frac_goal = (alpha - 2)/(2 * alpha) + 1/alpha * np.sqrt(1 - alpha * np.log(-2*(vol_frac - 1)))

d_eq, d_0 = parse_dist(options.d_dist)
r, r_0 = parse_dist(options.r_dist)
k, k_0 = parse_dist(options.k_dist)

fname = options.fname

# to avoid confusion with types:
m = np.int(options.m)
m_coarse = np.int(options.m_coarse)
L = np.float(options.L)
mc_steps = np.float(options.mc_steps)
vol_frac_goal = np.double(options.vol_frac_goal)
tau = np.double(options.tau)
nr_tries = np.int(options.nr_tries)
delta_x = d_0/float(m)
M = np.int(m * L / d_0)
M_coarse = np.int(m_coarse * L / d_0)

idelta = M
idelta_coarse = M_coarse
if options.delta:
    idelta = np.int(M * options.delta / L)
    idelta_coarse = np.int(M_coarse * options.delta / L)

trunc_triangles = ccb.prepare_triangles(vol_frac_goal, L, r, k, d_eq)
# trunc_triangles = trunc_triangles[:1]
# trunc_triangles[0].rot_matrix = np.eye(3)
# trunc_triangles[0].rot_matrix_tr = np.eye(3)
# trunc_triangles[0].midpoint = np.array([2., 2., 2.])

# Sort triangles w.r.t. volume, so that large triangles are added to the box first (better packing)
trunc_triangles.sort(key=lambda x: x.volume, reverse=True)
print('Prepared', len(trunc_triangles), 'triangles')

if options.use_potential:
    ccb.optimize_midpoints(L, trunc_triangles)

if m_coarse == m:
    grain_ids, overlaps, voxel_indices = ccb_c.populate_voxels(M, L, trunc_triangles, nr_tries, idelta, 1.0)
else:
    if nr_tries > 0:
        # Optimization: Use coarser grid for packing, then insert packed grains into fine grid
        # No need to get the return values, trunc_triangles
        ccb_c.populate_voxels(M_coarse, L, trunc_triangles, nr_tries, idelta_coarse, 1.0)
    grain_ids, overlaps, voxel_indices = ccb_c.populate_voxels(M, L, trunc_triangles, 1, 0, 1.0)

if mc_steps > 0:
    start_time = time.time()
    # Do Potts on coarse grid first for an improved initial guess.
    M_coarseMC = M//2
    grain_ids_coarse, overlaps_coarse, voxel_indices_coarse = ccb_c.populate_voxels(M_coarseMC, L, trunc_triangles, 0, 0, 1.0)
    _, gb_voxels_coarse, _ = ccb_c.calc_surface_prop(M_coarseMC, grain_ids_coarse)
    ccb_c.make_mcp_bound(M_coarseMC, grain_ids_coarse, gb_voxels_coarse, overlaps_coarse, voxel_indices_coarse,
                         np.int(mc_steps * M_coarseMC**4), tau)

    # Copy over that solution to the overlap regions of the fine grid as a starting point
    M2 = M**2
    i = np.nonzero(overlaps)[0]
    iz = i // M2
    iy = (i - iz*M2) // M
    ix = i - iz*M2 - iy*M
    cix = ix * M_coarseMC // M
    ciy = iy * M_coarseMC // M
    ciz = iz * M_coarseMC // M
    ci = cix + ciy*M_coarseMC + ciz*M_coarseMC**2
    gid = grain_ids_coarse[ci]
    # Could use a Cython implementation for efficiency.
    for ii, g in zip(i, gid):
        if g != grain_ids[ii] and np.searchsorted(voxel_indices[g-2], ii) < len(voxel_indices[g-2]):
            grain_ids[ii] = g
    # This might change a few voxels to a value that they shouldn't obtain, but it's barely noticeable
    # grain_ids_1[i] = grain_ids_coarse[ci]
    _, gb_voxels, _ = ccb_c.calc_surface_prop(M, grain_ids)
    # and run the full resolution MCP:
    ccb_c.make_mcp_bound(M, grain_ids, gb_voxels, overlaps, voxel_indices, np.int(mc_steps * M ** 4), tau)
    print('Potts model took {} seconds'.format(np.str(time.time() - start_time)))

if options.stray_cleanup:
    start_time = time.time()
    ccb_c.stray_cleanup(M, grain_ids)
    print('Stray voxel cleanup took {} seconds'.format(np.str(time.time() - start_time)))

surface_voxels, gb_voxels, interface_voxels = ccb_c.calc_surface_prop(M, grain_ids)
phases, good_voxels, euler_angles = ccb_c.calc_grain_prop(M, grain_ids, trunc_triangles)

phase_volumes = np.bincount(phases)
vol_frac_WC = phase_volumes[2] / np.float(M ** 3)
vol_frac_Co = 1 - vol_frac_WC
mass_frac_WC = ccb.mass_fraction(vol_frac_WC)

sum_gb_voxels = np.sum(gb_voxels)
contiguity = sum_gb_voxels / np.float(sum_gb_voxels + np.sum(interface_voxels))

print('Contiguity {:5f}, Co volume frac {:.5f}, mass frac {:.5f}'.format(
    contiguity, 1 - vol_frac_WC, ccb.mass_fraction(vol_frac_WC)))

ccb.write_dream3d(fname, 3 * [M], 3 * [delta_x], trunc_triangles, grain_ids, phases, good_voxels,
                  euler_angles, surface_voxels, gb_voxels, interface_voxels, overlaps)

with open(fname + '_trunc_triangles.data', 'wb') as f:
    pickle.dump([t.rot_matrix for t in trunc_triangles], f)

# Saving grain volume data
if False:
    grain_volumes = np.bincount(grain_ids)
    d_eq = ccb.volume_to_eq_d(grain_volumes[2:] * delta_x ** 3)
    # np.savetxt(fname + '_d_orig.txt', [t.d_eq for t in trunc_triangles])
    np.savetxt(fname + '_d.txt', d_eq)
    # Plot initial and final distributions
    import matplotlib.pyplot as plt
    plt.hist(np.array([t.d_eq for t in trunc_triangles]), alpha=0.5, bins=15, normed=True, label='Initial')
    plt.hist(d_eq, alpha=0.5, bins=15, normed=True, label='Final')
    plt.legend(loc='upper right')
    plt.show()


'''
Some functions to fit a random forest
'''

import sklearn.ensemble
import pandas
import progressbar

bar = progressbar.ProgressBar()

def test_max_features(max_features):
    if (max_features not in ['sqrt', 'auto', 'log2', None]):
        try:
            max_features = int(max_features)
        except ValueError:
            print("max_features has to be an integer or one of 'sqrt', 'auto', 'log2' or None.")
            raise
    return max_features

def learn(X,y, n_trees = 10, criterion = 'entropy', max_features = "sqrt", max_depth = None, min_samples_split = 2, min_samples_leaf = 1, min_weight_fraction_leaf = 0, max_leaf_nodes = None, min_impurity_split = 1e-7, bootstrap = False, oob_score = False, n_jobs = 10, random_state = None, warm_start = False, class_weight = 'balanced_subsample'):
    rf = sklearn.ensemble.RandomForestClassifier(n_estimators = n_trees, \
                                                criterion = criterion, \
                                                max_features = max_features, \
                                                max_depth = max_depth, \
                                                min_samples_split = min_samples_split, \
                                                min_samples_leaf = min_samples_leaf, \
                                                min_weight_fraction_leaf = min_weight_fraction_leaf, \
                                                max_leaf_nodes = max_leaf_nodes, \
                                                min_impurity_split = min_impurity_split, \
                                                bootstrap = bootstrap, \
                                                oob_score = oob_score, \
                                                n_jobs = n_jobs, \
                                                random_state = random_state, \
                                                warm_start = warm_start, \
                                                class_weight = class_weight, \
                                                verbose = 1
                                                )
    rf.fit(X, y)
    return rf

def importance(rf, kmers):
    importance = rf.estimators_[0].feature_importances_
    for est in bar(rf.estimators_[1:]):
        importance += est.feature_importances_
    importance = importance/rf.n_estimators
    d = {"kmer": kmers,
        "importance": importance}
    d = pandas.DataFrame(d)
    d = d.sort_values(by = "importance", ascending = 0)
    d = d.loc[d.importance > 0]
    return d


# -*- coding: utf-8 -*-
#
# cross_check_test_mip_corrdet.py
#
# This file is part of NEST.
#
# Copyright (C) 2004 The NEST Initiative
#
# NEST is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# NEST is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with NEST.  If not, see <http://www.gnu.org/licenses/>.

# Script to check correlation_detector.

# Calculates spike cross correlation function of both spike trains in
# spike_detector-0-0-3.gdf. The file is generated after running the
# testscript testsuite/unittests/test_mip_corrdet.sli
#
# Author: Helias
# Date: 08-04-07
#

from scipy import *
from matplotlib.pylab import * # for plot

# Auto- and crosscorrelation functions for spike trains.
#
# A time bin of size tbin is centered around the time difference it
# represents If the correlation function is calculated for tau in
# [-tau_max, tau_max], the pair events contributing to the left-most
# bin are those for which tau in [-tau_max-tbin/2, tau_max+tbin/2) and
# so on.

# correlate two spike trains with each other
# assumes spike times to be ordered in time
# tau > 0 means spike2 is later than spike1
#
# tau_max:     maximum time lag in ms correlation function
# tbin:   bin size
# spike1: first spike train [tspike...]
# spike2: second spike train [tspike...]
#
def corr_spikes_sorted(spike1, spike2, tbin, tau_max, h):

  tau_max_i = int(tau_max/h)
  tbin_i = int(tbin/h)

  cross = zeros(int(2*tau_max_i/tbin_i+1), 'd')

  j0 = 0

  for spki in spike1:
    j = j0
    while j < len(spike2) and spike2[j] - spki < -tau_max_i - tbin_i/2.0:
      j += 1
    j0 = j

    while j < len(spike2) and spike2[j] - spki < tau_max_i + tbin_i/2.0:
      cross[int((spike2[j] - spki + tau_max_i + 0.5*tbin_i)/tbin_i)] += 1.0
      j += 1

  return cross

def main():

  # resolution
  h = 0.1
  tau_max = 100.0 # ms correlation window
  t_bin = 10.0 # ms bin size

  # read input from spike detector
  spikes = load('spike_detector-0-0-3.gdf')

  sp1 = spikes[find(spikes[:,0] == 4), 1]
  sp2 = spikes[find(spikes[:,0] == 5), 1]

  cross = corr_spikes_sorted(sp1, sp2, t_bin, tau_max, h)

  print cross
  print sum(cross)

main()


# -*- coding: utf-8 -*-

import os
import seaborn

seaborn.set()
colors = seaborn.color_palette()

import utils

# For 3D plotting we need to import some extra stuff
from mpl_toolkits.mplot3d import Axes3D

# First create two random vectors in 3 dimensional space
v1 = rand(3, 1)
v2 = rand(3, 1)

# And scale them to unit length
v1 = v1 / norm(v1)
v2 = v2 / norm(v2)

# Plot the vectors
o = zeros(3)  # origin

# We'll use the object oriented plotting interface
f = figure(figsize=(8, 8))
ax = f.add_subplot(111, projection="3d", axisbg="white")
ax.plot(*[[o[i], v1[i]] for i in range(3)], linewidth=3, label="vector1")
ax.plot(*[[o[i], v2[i]] for i in range(3)], linewidth=3, label="vector2")
for axisl in ["x", "y", "z"]:
    getattr(ax, "set_%slabel" % axisl)(axisl)  # Here's a fun trick
legend()

f = figure(figsize=(8, 8))
ax = f.add_subplot(111, projection="3d", axisbg="white")
ax.plot(*[[o[i], v1[i]] for i in range(3)], linewidth=3, label="vector1")
ax.plot(*[[o[i], v2[i]] for i in range(3)], linewidth=3, label="vector2")
for axisl in ["x", "y", "z"]:
    getattr(ax, "set_%slabel" % axisl)(axisl)  # Here's a fun trick
legend()

for i in range(100):
    # generate a point that is a weighted sum of the 2 vectors
    w1 = randn(1)
    w2 = randn(1)
    point = w1 * v1 + w2 * v2
    ax.plot(*point, marker=".", color="k")

# We can find a vector that is orthogonal to the plane defined by v1 and v2
# by taking the vector cross product.  See the wikipedia page for a
# definition of cross product
# Must be right shape for cross()
v3 = cross(v1.reshape(1, 3), v2.reshape(1, 3)).squeeze()
ax.plot(*[[o[i], v3[i]] for i in range(3)],
        linewidth=3, label="orthogonal vector")
legend()

print(v3[0] * v1[0] + v3[1] * v1[1] + v3[2] * v1[2])
print(dot(v3, v1))

theta = arccos(dot(v2.T, v1)).squeeze()
# and radians can be converted to degrees
theta_deg = theta * (180 / pi)
print(theta, theta_deg)

os.system("pause")


"""
Contains a class for managing paths (polylines).
"""

import math
from weakref import WeakValueDictionary

import numpy as np
from numpy import ma

from matplotlib._path import point_in_path, get_path_extents, \
    point_in_path_collection, get_path_collection_extents, \
    path_in_path, path_intersects_path, convert_path_to_polygons
from matplotlib.cbook import simple_linear_interpolation

class Path(object):
    """
    :class:`Path` represents a series of possibly disconnected,
    possibly closed, line and curve segments.

    The underlying storage is made up of two parallel numpy arrays:
      - *vertices*: an Nx2 float array of vertices
      - *codes*: an N-length uint8 array of vertex types

    These two arrays always have the same length in the first
    dimension.  For example, to represent a cubic curve, you must
    provide three vertices as well as three codes ``CURVE3``.

    The code types are:

       - ``STOP``   :  1 vertex (ignored)
           A marker for the end of the entire path (currently not
           required and ignored)

       - ``MOVETO`` :  1 vertex
            Pick up the pen and move to the given vertex.

       - ``LINETO`` :  1 vertex
            Draw a line from the current position to the given vertex.

       - ``CURVE3`` :  1 control point, 1 endpoint
          Draw a quadratic Bezier curve from the current position,
          with the given control point, to the given end point.

       - ``CURVE4`` :  2 control points, 1 endpoint
          Draw a cubic Bezier curve from the current position, with
          the given control points, to the given end point.

       - ``CLOSEPOLY`` : 1 vertex (ignored)
          Draw a line segment to the start point of the current
          polyline.

    Users of Path objects should not access the vertices and codes
    arrays directly.  Instead, they should use :meth:`iter_segments`
    to get the vertex/code pairs.  This is important, since many
    :class:`Path` objects, as an optimization, do not store a *codes*
    at all, but have a default one provided for them by
    :meth:`iter_segments`.

    Note also that the vertices and codes arrays should be treated as
    immutable -- there are a number of optimizations and assumptions
    made up front in the constructor that will not change when the
    data changes.
    """

    # Path codes
    STOP      = 0 # 1 vertex
    MOVETO    = 1 # 1 vertex
    LINETO    = 2 # 1 vertex
    CURVE3    = 3 # 2 vertices
    CURVE4    = 4 # 3 vertices
    CLOSEPOLY = 5 # 1 vertex

    NUM_VERTICES = [1, 1, 1, 2, 3, 1]

    code_type = np.uint8

    def __init__(self, vertices, codes=None):
        """
        Create a new path with the given vertices and codes.

        *vertices* is an Nx2 numpy float array, masked array or Python
        sequence.

        *codes* is an N-length numpy array or Python sequence of type
        :attr:`matplotlib.path.Path.code_type`.

        These two arrays must have the same length in the first
        dimension.

        If *codes* is None, *vertices* will be treated as a series of
        line segments.

        If *vertices* contains masked values, they will be converted
        to NaNs which are then handled correctly by the Agg
        PathIterator and other consumers of path data, such as
        :meth:`iter_segments`.
        """
        if ma.isMaskedArray(vertices):
            vertices = vertices.astype(np.float_).filled(np.nan)
        else:
            vertices = np.asarray(vertices, np.float_)

        if codes is not None:
            codes = np.asarray(codes, self.code_type)
            assert codes.ndim == 1
            assert len(codes) == len(vertices)

        assert vertices.ndim == 2
        assert vertices.shape[1] == 2

        self.should_simplify = (len(vertices) >= 128 and
                                (codes is None or np.all(codes <= Path.LINETO)))
        self.has_nonfinite = not np.isfinite(vertices).all()
        self.codes = codes
        self.vertices = vertices

    #@staticmethod
    def make_compound_path(*args):
        """
        (staticmethod) Make a compound path from a list of Path
        objects.  Only polygons (not curves) are supported.
        """
        for p in args:
            assert p.codes is None

        lengths = [len(x) for x in args]
        total_length = sum(lengths)

        vertices = np.vstack([x.vertices for x in args])
        vertices.reshape((total_length, 2))

        codes = Path.LINETO * np.ones(total_length)
        i = 0
        for length in lengths:
            codes[i] = Path.MOVETO
            i += length

        return Path(vertices, codes)
    make_compound_path = staticmethod(make_compound_path)

    def __repr__(self):
        return "Path(%s, %s)" % (self.vertices, self.codes)

    def __len__(self):
        return len(self.vertices)

    def iter_segments(self, simplify=None):
        """
        Iterates over all of the curve segments in the path.  Each
        iteration returns a 2-tuple (*vertices*, *code*), where
        *vertices* is a sequence of 1 - 3 coordinate pairs, and *code* is
        one of the :class:`Path` codes.

        If *simplify* is provided, it must be a tuple (*width*,
        *height*) defining the size of the figure, in native units
        (e.g. pixels or points).  Simplification implies both removing
        adjacent line segments that are very close to parallel, and
        removing line segments outside of the figure.  The path will
        be simplified *only* if :attr:`should_simplify` is True, which
        is determined in the constructor by this criteria:

           - No curves
           - More than 128 vertices
        """
        vertices = self.vertices
        if not len(vertices):
            return

        codes        = self.codes
        len_vertices = len(vertices)
        isfinite     = np.isfinite

        NUM_VERTICES = self.NUM_VERTICES
        MOVETO       = self.MOVETO
        LINETO       = self.LINETO
        CLOSEPOLY    = self.CLOSEPOLY
        STOP         = self.STOP

        if simplify is not None and self.should_simplify:
            polygons = self.to_polygons(None, *simplify)
            for vertices in polygons:
                yield vertices[0], MOVETO
                for v in vertices[1:]:
                    yield v, LINETO
        elif codes is None:
            if self.has_nonfinite:
                next_code = MOVETO
                for v in vertices:
                    if np.isfinite(v).all():
                        yield v, next_code
                        next_code = LINETO
                    else:
                        next_code = MOVETO
            else:
                yield vertices[0], MOVETO
                for v in vertices[1:]:
                    yield v, LINETO
        else:
            i = 0
            was_nan = False
            while i < len_vertices:
                code = codes[i]
                if code == CLOSEPOLY:
                    yield [], code
                    i += 1
                elif code == STOP:
                    return
                else:
                    num_vertices = NUM_VERTICES[int(code)]
                    curr_vertices = vertices[i:i+num_vertices].flatten()
                    if not isfinite(curr_vertices).all():
                        was_nan = True
                    elif was_nan:
                        yield curr_vertices[-2:], MOVETO
                        was_nan = False
                    else:
                        yield curr_vertices, code
                    i += num_vertices

    def transformed(self, transform):
        """
        Return a transformed copy of the path.

        .. seealso::
            :class:`matplotlib.transforms.TransformedPath`:
                A specialized path class that will cache the
                transformed result and automatically update when the
                transform changes.
        """
        return Path(transform.transform(self.vertices), self.codes)

    def contains_point(self, point, transform=None):
        """
        Returns *True* if the path contains the given point.

        If *transform* is not *None*, the path will be transformed
        before performing the test.
        """
        if transform is not None:
            transform = transform.frozen()
        return point_in_path(point[0], point[1], self, transform)

    def contains_path(self, path, transform=None):
        """
        Returns *True* if this path completely contains the given path.

        If *transform* is not *None*, the path will be transformed
        before performing the test.
        """
        if transform is not None:
            transform = transform.frozen()
        return path_in_path(self, None, path, transform)

    def get_extents(self, transform=None):
        """
        Returns the extents (*xmin*, *ymin*, *xmax*, *ymax*) of the
        path.

        Unlike computing the extents on the *vertices* alone, this
        algorithm will take into account the curves and deal with
        control points appropriately.
        """
        from transforms import Bbox
        if transform is not None:
            transform = transform.frozen()
        return Bbox(get_path_extents(self, transform))

    def intersects_path(self, other, filled=True):
        """
        Returns *True* if this path intersects another given path.

        *filled*, when True, treats the paths as if they were filled.
        That is, if one path completely encloses the other,
        :meth:`intersects_path` will return True.
        """
        return path_intersects_path(self, other, filled)

    def intersects_bbox(self, bbox, filled=True):
        """
        Returns *True* if this path intersects a given
        :class:`~matplotlib.transforms.Bbox`.

        *filled*, when True, treats the path as if it was filled.
        That is, if one path completely encloses the other,
        :meth:`intersects_path` will return True.
        """
        from transforms import BboxTransformTo
        rectangle = self.unit_rectangle().transformed(
            BboxTransformTo(bbox))
        result = self.intersects_path(rectangle, filled)
        return result

    def interpolated(self, steps):
        """
        Returns a new path resampled to length N x steps.  Does not
        currently handle interpolating curves.
        """
        vertices = simple_linear_interpolation(self.vertices, steps)
        codes = self.codes
        if codes is not None:
            new_codes = Path.LINETO * np.ones(((len(codes) - 1) * steps + 1, ))
            new_codes[0::steps] = codes
        else:
            new_codes = None
        return Path(vertices, new_codes)

    def to_polygons(self, transform=None, width=0, height=0):
        """
        Convert this path to a list of polygons.  Each polygon is an
        Nx2 array of vertices.  In other words, each polygon has no
        ``MOVETO`` instructions or curves.  This is useful for
        displaying in backends that do not support compound paths or
        Bezier curves, such as GDK.

        If *width* and *height* are both non-zero then the lines will
        be simplified so that vertices outside of (0, 0), (width,
        height) will be clipped.
        """
        if len(self.vertices) == 0:
            return []

        if transform is not None:
            transform = transform.frozen()

        if self.codes is None and (width == 0 or height == 0):
            if transform is None:
                return [self.vertices]
            else:
                return [transform.transform(self.vertices)]

        # Deal with the case where there are curves and/or multiple
        # subpaths (using extension code)
        return convert_path_to_polygons(self, transform, width, height)

    _unit_rectangle = None
    #@classmethod
    def unit_rectangle(cls):
        """
        (staticmethod) Returns a :class:`Path` of the unit rectangle
        from (0, 0) to (1, 1).
        """
        if cls._unit_rectangle is None:
            cls._unit_rectangle = \
                Path([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0]])
        return cls._unit_rectangle
    unit_rectangle = classmethod(unit_rectangle)

    _unit_regular_polygons = WeakValueDictionary()
    #@classmethod
    def unit_regular_polygon(cls, numVertices):
        """
        (staticmethod) Returns a :class:`Path` for a unit regular
        polygon with the given *numVertices* and radius of 1.0,
        centered at (0, 0).
        """
        if numVertices <= 16:
            path = cls._unit_regular_polygons.get(numVertices)
        else:
            path = None
        if path is None:
            theta = (2*np.pi/numVertices *
                     np.arange(numVertices + 1).reshape((numVertices + 1, 1)))
            # This initial rotation is to make sure the polygon always
            # "points-up"
            theta += np.pi / 2.0
            verts = np.concatenate((np.cos(theta), np.sin(theta)), 1)
            path = Path(verts)
            cls._unit_regular_polygons[numVertices] = path
        return path
    unit_regular_polygon = classmethod(unit_regular_polygon)

    _unit_regular_stars = WeakValueDictionary()
    #@classmethod
    def unit_regular_star(cls, numVertices, innerCircle=0.5):
        """
        (staticmethod) Returns a :class:`Path` for a unit regular star
        with the given numVertices and radius of 1.0, centered at (0,
        0).
        """
        if numVertices <= 16:
            path = cls._unit_regular_stars.get((numVertices, innerCircle))
        else:
            path = None
        if path is None:
            ns2 = numVertices * 2
            theta = (2*np.pi/ns2 * np.arange(ns2 + 1))
            # This initial rotation is to make sure the polygon always
            # "points-up"
            theta += np.pi / 2.0
            r = np.ones(ns2 + 1)
            r[1::2] = innerCircle
            verts = np.vstack((r*np.cos(theta), r*np.sin(theta))).transpose()
            path = Path(verts)
            cls._unit_regular_polygons[(numVertices, innerCircle)] = path
        return path
    unit_regular_star = classmethod(unit_regular_star)

    #@classmethod
    def unit_regular_asterisk(cls, numVertices):
        """
        (staticmethod) Returns a :class:`Path` for a unit regular
        asterisk with the given numVertices and radius of 1.0,
        centered at (0, 0).
        """
        return cls.unit_regular_star(numVertices, 0.0)
    unit_regular_asterisk = classmethod(unit_regular_asterisk)

    _unit_circle = None
    #@classmethod
    def unit_circle(cls):
        """
        (staticmethod) Returns a :class:`Path` of the unit circle.
        The circle is approximated using cubic Bezier curves.  This
        uses 8 splines around the circle using the approach presented
        here:

          Lancaster, Don.  `Approximating a Circle or an Ellipse Using Four
          Bezier Cubic Splines <http://www.tinaja.com/glib/ellipse4.pdf>`_.
        """
        if cls._unit_circle is None:
            MAGIC = 0.2652031
            SQRTHALF = np.sqrt(0.5)
            MAGIC45 = np.sqrt((MAGIC*MAGIC) / 2.0)

            vertices = np.array(
                [[0.0, -1.0],

                 [MAGIC, -1.0],
                 [SQRTHALF-MAGIC45, -SQRTHALF-MAGIC45],
                 [SQRTHALF, -SQRTHALF],

                 [SQRTHALF+MAGIC45, -SQRTHALF+MAGIC45],
                 [1.0, -MAGIC],
                 [1.0, 0.0],

                 [1.0, MAGIC],
                 [SQRTHALF+MAGIC45, SQRTHALF-MAGIC45],
                 [SQRTHALF, SQRTHALF],

                 [SQRTHALF-MAGIC45, SQRTHALF+MAGIC45],
                 [MAGIC, 1.0],
                 [0.0, 1.0],

                 [-MAGIC, 1.0],
                 [-SQRTHALF+MAGIC45, SQRTHALF+MAGIC45],
                 [-SQRTHALF, SQRTHALF],

                 [-SQRTHALF-MAGIC45, SQRTHALF-MAGIC45],
                 [-1.0, MAGIC],
                 [-1.0, 0.0],

                 [-1.0, -MAGIC],
                 [-SQRTHALF-MAGIC45, -SQRTHALF+MAGIC45],
                 [-SQRTHALF, -SQRTHALF],

                 [-SQRTHALF+MAGIC45, -SQRTHALF-MAGIC45],
                 [-MAGIC, -1.0],
                 [0.0, -1.0],

                 [0.0, -1.0]],
                np.float_)

            codes = cls.CURVE4 * np.ones(26)
            codes[0] = cls.MOVETO
            codes[-1] = cls.CLOSEPOLY

            cls._unit_circle = Path(vertices, codes)
        return cls._unit_circle
    unit_circle = classmethod(unit_circle)

    #@classmethod
    def arc(cls, theta1, theta2, n=None, is_wedge=False):
        """
        (staticmethod) Returns an arc on the unit circle from angle
        *theta1* to angle *theta2* (in degrees).

        If *n* is provided, it is the number of spline segments to make.
        If *n* is not provided, the number of spline segments is
        determined based on the delta between *theta1* and *theta2*.

           Masionobe, L.  2003.  `Drawing an elliptical arc using
           polylines, quadratic or cubic Bezier curves
           <http://www.spaceroots.org/documents/ellipse/index.html>`_.
        """
        # degrees to radians
        theta1 *= np.pi / 180.0
        theta2 *= np.pi / 180.0

        twopi  = np.pi * 2.0
        halfpi = np.pi * 0.5

        eta1 = np.arctan2(np.sin(theta1), np.cos(theta1))
        eta2 = np.arctan2(np.sin(theta2), np.cos(theta2))
        eta2 -= twopi * np.floor((eta2 - eta1) / twopi)
        if (theta2 - theta1 > np.pi) and (eta2 - eta1 < np.pi):
            eta2 += twopi

        # number of curve segments to make
        if n is None:
            n = int(2 ** np.ceil((eta2 - eta1) / halfpi))
        if n < 1:
            raise ValueError("n must be >= 1 or None")

        deta = (eta2 - eta1) / n
        t = np.tan(0.5 * deta)
        alpha = np.sin(deta) * (np.sqrt(4.0 + 3.0 * t * t) - 1) / 3.0

        steps = np.linspace(eta1, eta2, n + 1, True)
        cos_eta = np.cos(steps)
        sin_eta = np.sin(steps)

        xA = cos_eta[:-1]
        yA = sin_eta[:-1]
        xA_dot = -yA
        yA_dot = xA

        xB = cos_eta[1:]
        yB = sin_eta[1:]
        xB_dot = -yB
        yB_dot = xB

        if is_wedge:
            length = n * 3 + 4
            vertices = np.zeros((length, 2), np.float_)
            codes = Path.CURVE4 * np.ones((length, ), Path.code_type)
            vertices[1] = [xA[0], yA[0]]
            codes[0:2] = [Path.MOVETO, Path.LINETO]
            codes[-2:] = [Path.LINETO, Path.CLOSEPOLY]
            vertex_offset = 2
            end = length - 2
        else:
            length = n * 3 + 1
            vertices = np.zeros((length, 2), np.float_)
            codes = Path.CURVE4 * np.ones((length, ), Path.code_type)
            vertices[0] = [xA[0], yA[0]]
            codes[0] = Path.MOVETO
            vertex_offset = 1
            end = length

        vertices[vertex_offset  :end:3, 0] = xA + alpha * xA_dot
        vertices[vertex_offset  :end:3, 1] = yA + alpha * yA_dot
        vertices[vertex_offset+1:end:3, 0] = xB - alpha * xB_dot
        vertices[vertex_offset+1:end:3, 1] = yB - alpha * yB_dot
        vertices[vertex_offset+2:end:3, 0] = xB
        vertices[vertex_offset+2:end:3, 1] = yB

        return Path(vertices, codes)
    arc = classmethod(arc)

    #@classmethod
    def wedge(cls, theta1, theta2, n=None):
        """
        (staticmethod) Returns a wedge of the unit circle from angle
        *theta1* to angle *theta2* (in degrees).

        If *n* is provided, it is the number of spline segments to make.
        If *n* is not provided, the number of spline segments is
        determined based on the delta between *theta1* and *theta2*.
        """
        return cls.arc(theta1, theta2, n, True)
    wedge = classmethod(wedge)

_get_path_collection_extents = get_path_collection_extents
def get_path_collection_extents(*args):
    """
    Given a sequence of :class:`Path` objects, returns the bounding
    box that encapsulates all of them.
    """
    from transforms import Bbox
    if len(args[1]) == 0:
        raise ValueError("No paths provided")
    return Bbox.from_extents(*_get_path_collection_extents(*args))


"""
The :mod:`sklearn.linear_model` module implements generalized linear models. It
includes Ridge regression, Bayesian Regression, Lasso and Elastic Net
estimators computed with Least Angle Regression and coordinate descent. It also
implements Stochastic Gradient Descent related algorithms.
"""

# See http://scikit-learn.sourceforge.net/modules/sgd.html and
# http://scikit-learn.sourceforge.net/modules/linear_model.html for
# complete documentation.

from .base import LinearRegression

from .bayes import BayesianRidge, ARDRegression
from .least_angle import (Lars, LassoLars, lars_path, LarsCV, LassoLarsCV,
                          LassoLarsIC)
from .coordinate_descent import (Lasso, ElasticNet, LassoCV, ElasticNetCV,
                                 lasso_path, enet_path, MultiTaskLasso,
                                 MultiTaskElasticNet, MultiTaskElasticNetCV,
                                 MultiTaskLassoCV)
from .huber import HuberRegressor
from .sgd_fast import Hinge, Log, ModifiedHuber, SquaredLoss, Huber
from .stochastic_gradient import SGDClassifier, SGDRegressor
from .ridge import (Ridge, RidgeCV, RidgeClassifier, RidgeClassifierCV,
                    ridge_regression)
from .logistic import (LogisticRegression, LogisticRegressionCV,
                       logistic_regression_path)
from .omp import (orthogonal_mp, orthogonal_mp_gram, OrthogonalMatchingPursuit,
                  OrthogonalMatchingPursuitCV)
from .passive_aggressive import PassiveAggressiveClassifier
from .passive_aggressive import PassiveAggressiveRegressor
from .perceptron import Perceptron
from .randomized_l1 import (RandomizedLasso, RandomizedLogisticRegression,
                            lasso_stability_path)
from .ransac import RANSACRegressor
from .theil_sen import TheilSenRegressor

__all__ = ['ARDRegression',
           'BayesianRidge',
           'ElasticNet',
           'ElasticNetCV',
           'Hinge',
           'HuberRegressor',
           'Lars',
           'LarsCV',
           'Lasso',
           'LassoCV',
           'LassoLars',
           'LassoLarsCV',
           'LassoLarsIC',
           'LinearRegression',
           'Log',
           'LogisticRegression',
           'LogisticRegressionCV',
           'ModifiedHuber',
           'MultiTaskElasticNet',
           'MultiTaskElasticNetCV',
           'MultiTaskLasso',
           'MultiTaskLassoCV',
           'OrthogonalMatchingPursuit',
           'OrthogonalMatchingPursuitCV',
           'PassiveAggressiveClassifier',
           'PassiveAggressiveRegressor',
           'Perceptron',
           'RandomizedLasso',
           'RandomizedLogisticRegression',
           'Ridge',
           'RidgeCV',
           'RidgeClassifier',
           'RidgeClassifierCV',
           'SGDClassifier',
           'SGDRegressor',
           'SquaredLoss',
           'TheilSenRegressor',
           'enet_path',
           'lars_path',
           'lasso_path',
           'lasso_stability_path',
           'logistic_regression_path',
           'orthogonal_mp',
           'orthogonal_mp_gram',
           'ridge_regression',
           'RANSACRegressor']


train_ratio=0.9
use_dict=True
use_scaler=False
init_emb=False
split_contins=True
samp_size = 100000
#samp_size = 0

import math, keras, datetime, pandas as pd, numpy as np, keras.backend as K
import matplotlib.pyplot as plt, xgboost, operator, random, pickle, os
from sklearn_pandas import DataFrameMapper
from sklearn.preprocessing import LabelEncoder, Imputer, StandardScaler
from keras.models import Model
from keras.layers import merge, Input
from keras.layers.core import Dense, Activation, Reshape, Flatten, Dropout
from keras.layers.embeddings import Embedding
from keras.optimizers import Adam
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l2
from keras import initializations
np.set_printoptions(4)

cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))

os.chdir('data/rossman')
cat_var_dict = {'Store': 50, 'DayOfWeek': 6, 'Year': 2, 'Month': 6,
    'Day': 10, 'StateHoliday': 3, 'CompetitionMonthsOpen': 2,
    'Promo2Weeks': 1, 'StoreType': 2, 'Assortment': 3, 'PromoInterval': 3,
    'CompetitionOpenSinceYear': 4, 'Promo2SinceYear': 4, 'State': 6,
    'Week': 2, 'Events': 4, 'Promo_fw': 1,
    'Promo_bw': 1, 'StateHoliday_fw': 1,
    'StateHoliday_bw': 1, 'SchoolHoliday_fw': 1,
    'SchoolHoliday_bw': 1}

cats, contins= [o for n,o in np.load('vars.npz').items()]
y = np.load('deps.npz').items()[0][1]

if samp_size != 0:
    np.random.seed(42)
    idxs = sorted(np.random.choice(len(y), samp_size, replace=False))
    cats= cats[idxs]
    contins= contins[idxs]
    y= y[idxs]

n=len(y)
train_size = int(n*train_ratio)

contins_trn_orig, contins_val_orig = contins[:train_size], contins[train_size:]
cats_trn, cats_val = cats[:train_size], cats[train_size:]
y_trn, y_val = y[:train_size], y[train_size:]

contin_map_fit = pickle.load(open('contin_maps.pickle', 'rb'))
cat_map_fit = pickle.load(open('cat_maps.pickle', 'rb'))

def cat_map_info(feat): return feat[0], len(feat[1].classes_)

co_enc = StandardScaler().fit(contins_trn_orig)
tf_contins_trn = co_enc.transform(contins_trn_orig)
tf_contins_val = co_enc.transform(contins_val_orig)

"""
def rmspe(y_pred, targ = y_valid_orig):
    return math.sqrt(np.square((targ - y_pred)/targ).mean())
def log_max_inv(preds, mx = max_log_y): return np.exp(preds * mx)
def normalize_inv(preds): return preds * ystd + ymean
"""

def split_cols(arr): return np.hsplit(arr,arr.shape[1])

def emb_init(shape, name=None):
    return initializations.uniform(shape, scale=0.6/shape[1], name=name)

def get_emb(feat):
    name, c = cat_map_info(feat)
    if use_dict:
        c2 = cat_var_dict[name]
    else:
        c2 = (c+2)//3
        if c2>50: c2=50
    inp = Input((1,), dtype='int64', name=name+'_in')
    if init_emb:
        u = Flatten(name=name+'_flt')(Embedding(c, c2, input_length=1)(inp))
    else:
        u = Flatten(name=name+'_flt')(Embedding(c, c2, input_length=1, init=emb_init)(inp))
    return inp,u

def get_contin(feat):
    name = feat[0][0]
    inp = Input((1,), name=name+'_in')
    return inp, Dense(1, name=name+'_d')(inp)

def split_data():
    if split_contins:
        map_train = split_cols(cats_trn) + split_cols(contins_trn)
        map_valid = split_cols(cats_val) + split_cols(contins_val)
    else:
        map_train = split_cols(cats_trn) + [contins_trn]
        map_valid = split_cols(cats_val) + [contins_val]
    return (map_train, map_valid)

def get_contin_one():
    n_contin = contins_trn.shape[1]
    contin_inp = Input((n_contin,), name='contin')
    contin_out = BatchNormalization()(contin_inp)
    return contin_inp, contin_out

def train(model, map_train, map_valid,  bs=128, ne=10):
    return model.fit(map_train, y_trn, batch_size=bs, nb_epoch=ne,
                 verbose=0, validation_data=(map_valid, y_val))

def get_model():
    if split_contins:
        conts = [get_contin(feat) for feat in contin_map_fit.features]
        cont_out = [d for inp,d in conts]
        cont_inp = [inp for inp,d in conts]
    else:
        contin_inp, contin_out = get_contin_one()
        cont_out = [contin_out]
        cont_inp = [contin_inp]

    embs = [get_emb(feat) for feat in cat_map_fit.features]
    x = merge([emb for inp,emb in embs] + cont_out, mode='concat')

    x = Dropout(0.02)(x)
    x = Dense(1000, activation='relu', init='uniform')(x)
    x = Dense(500, activation='relu', init='uniform')(x)
    x = Dense(1, activation='sigmoid')(x)

    model = Model([inp for inp,emb in embs] + cont_inp, x)
    model.compile('adam', 'mean_absolute_error')
    #model.compile(Adam(), 'mse')
    return model

for split_contins in [True, False]:
    for use_dict in [True, False]:
        for use_scaler in [True, False]:
            for init_emb in [True, False]:
                print ({'split_contins':split_contins, 'use_dict':use_dict,
                       'use_scaler':use_scaler, 'init_emb':init_emb})
                if use_scaler:
                    contins_trn = tf_contins_trn
                    contins_val = tf_contins_val
                else:
                    contins_trn = contins_trn_orig
                    contins_val = contins_val_orig

                map_train, map_valid = split_data()
                model = get_model()
                hist = np.array(train(model, map_train, map_valid, 128, 10)
                                .history['val_loss'])
                print(hist)
                print(hist.min())


Machine Learning A-Z/Part 7 - Natural Language Processing/Section 36 - Natural Language Processing/natural_language_processing.py

# Natural Language Processing

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Restaurant_Reviews.tsv', delimiter = '\t', quoting = 3)

# Cleaning the texts
import re
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
corpus = []
for i in range(0, 1000):
    review = re.sub('[^a-zA-Z]', ' ', dataset['Review'][i])
    review = review.lower()
    review = review.split()
    ps = PorterStemmer()
    review = [ps.stem(word) for word in review if not word in set(stopwords.words('english'))]
    review = ' '.join(review)
    corpus.append(review)

# Creating the Bag of Words model
from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(max_features = 1500)
X = cv.fit_transform(corpus).toarray()
y = dataset.iloc[:, 1].values

# Splitting the dataset into the Training set and Test set
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.20, random_state = 0)

# Fitting Naive Bayes to the Training set
from sklearn.naive_bayes import GaussianNB
classifier = GaussianNB()
classifier.fit(X_train, y_train)

# Predicting the Test set results
y_pred = classifier.predict(X_test)

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)