""" Properties are objects that can be assigned as class level
attributes on Bokeh models, to provide automatic serialization
and validation.

For example, the following defines a model that has integer,
string, and list[float] properties::

    class Model(HasProps):
        foo = Int
        bar = String
        baz = List(Float)

The properties of this class can be initialized by specifying
keyword arguments to the initializer::

    m = Model(foo=10, bar="a str", baz=[1,2,3,4])

But also by setting the attributes on an instance::

    m.foo = 20

Attempts to set a property to a value of the wrong type will
result in a ``ValueError`` exception::

    >>> m.foo = 2.3
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
      File "/Users/bryan/work/bokeh/bokeh/properties.py", line 585, in __setattr__
        super(HasProps, self).__setattr__(name, value)
      File "/Users/bryan/work/bokeh/bokeh/properties.py", line 159, in __set__
        raise e
      File "/Users/bryan/work/bokeh/bokeh/properties.py", line 152, in __set__
        self.validate(value)
      File "/Users/bryan/work/bokeh/bokeh/properties.py", line 707, in validate
        (nice_join([ cls.__name__ for cls in self._underlying_type ]), value, type(value).__name__))
    ValueError: expected a value of type int8, int16, int32, int64 or int, got 2.3 of type float

Additionally, properties know how to serialize themselves,
to be understood by BokehJS.

"""
from __future__ import absolute_import, print_function

import re
import types
import difflib
import datetime
import dateutil.parser
import collections
from importlib import import_module
from copy import copy
from warnings import warn
import inspect
import logging
logger = logging.getLogger(__name__)

from six import integer_types, string_types, add_metaclass, iteritems
import numpy as np

from . import enums
from .util.string import nice_join

def field(name):
    ''' Convenience function do explicitly mark a field specification for
    a Bokeh model property.

    Args:
        name (str) : name of a data source field to reference for a property.

    Returns:
        dict : `{"field": name}`

    Note:
        This function is included for completeness. String values for
        property specifications are by default interpreted as field names.

    '''
    return dict(field=name)

def value(val):
    ''' Convenience function do explicitly mark a value specification for
    a Bokeh model property.

    Args:
        val (any) : a fixed value to specify for a property.

    Returns:
        dict : `{"value": name}`

    Note:
        String values for property specifications are by default interpreted
        as field names. This function is especially useful when you want to
        specify a fixed value with text properties.

    Example:

    .. code-block:: python

        # The following will take text values to render from a data source
        # column "text_column", but use a fixed value "12pt" for font size
        p.text("x", "y", text="text_column",
               text_font_size=value("12pt"), source=source)

    '''
    return dict(value=val)

bokeh_integer_types = (np.int8, np.int16, np.int32, np.int64) + integer_types

# used to indicate properties that are not set (vs null, None, etc)
class _NotSet(object):
    pass

class DeserializationError(Exception):
    pass

class Property(object):
    """ Base class for all type properties. """

    def __init__(self, default=None, help=None):
        """ This is how the descriptor is created in the class declaration """
        if isinstance(default, types.FunctionType): # aka. lazy value
            self.validate(default())
        else:
            self.validate(default)

        self._default = default
        self.__doc__ = help
        self.alternatives = []

        # This gets set by the class decorator at class creation time
        self.name = "unnamed"

    def __str__(self):
        return self.__class__.__name__

    @property
    def _name(self):
        return "_" + self.name

    @property
    def default(self):
        if not isinstance(self._default, types.FunctionType):
            return copy(self._default)
        else:
            value = self._default()
            self.validate(value)
            return value

    @classmethod
    def autocreate(cls, name=None):
        """ Called by the metaclass to create a
        new instance of this descriptor
        if the user just assigned it to a property without trailing
        parentheses.
        """
        return cls()

    def matches(self, new, old):
        # XXX: originally this code warned about not being able to compare values, but that
        # doesn't make sense, because most comparisons involving numpy arrays will fail with
        # ValueError exception, thus warning about inevitable.
        try:
            if new is None or old is None:
                return new is old           # XXX: silence FutureWarning from NumPy
            else:
                return new == old
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            logger.debug("could not compare %s and %s for property %s (Reason: %s)", new, old, self.name, e)
        return False

    def from_json(self, json, models=None):
        return json

    def transform(self, value):
        return value

    def validate(self, value):
        pass

    def is_valid(self, value):
        try:
            self.validate(value)
        except ValueError:
            return False
        else:
            return True

    def _get(self, obj):
        if not hasattr(obj, self._name):
            setattr(obj, self._name, self.default)
        return getattr(obj, self._name)

    def __get__(self, obj, owner=None):
        if obj is not None:
            return self._get(obj)
        elif owner is not None:
            return self
        else:
            raise ValueError("both 'obj' and 'owner' are None, don't know what to do")

    def __set__(self, obj, value):
        try:
            self.validate(value)
        except ValueError as e:
            for tp, converter in self.alternatives:
                if tp.is_valid(value):
                    value = converter(value)
                    break
            else:
                raise e
        else:
            value = self.transform(value)

        old = self.__get__(obj)
        obj._changed_vars.add(self.name)
        if self._name in obj.__dict__ and self.matches(value, old):
            return
        setattr(obj, self._name, value)
        obj._dirty = True
        if hasattr(obj, '_trigger'):
            if hasattr(obj, '_block_callbacks') and obj._block_callbacks:
                obj._callback_queue.append((self.name, old, value))
            else:
                obj._trigger(self.name, old, value)

    def __delete__(self, obj):
        if hasattr(obj, self._name):
            delattr(obj, self._name)

    @property
    def has_ref(self):
        return False

    def accepts(self, tp, converter):
        tp = ParameterizedProperty._validate_type_param(tp)
        self.alternatives.append((tp, converter))
        return self

    def __or__(self, other):
        return Either(self, other)

class Include(object):
    """ Include other properties from mixin Models, with a given prefix. """

    def __init__(self, delegate, help="", use_prefix=True):
        if not (isinstance(delegate, type) and issubclass(delegate, HasProps)):
            raise ValueError("expected a subclass of HasProps, got %r" % delegate)

        self.delegate = delegate
        self.help = help
        self.use_prefix = use_prefix

class MetaHasProps(type):
    def __new__(cls, class_name, bases, class_dict):
        names = set()
        names_with_refs = set()
        container_names = set()

        # First pre-process to handle all the Includes
        includes = {}
        removes = set()
        for name, prop in class_dict.items():
            if not isinstance(prop, Include):
                continue

            delegate = prop.delegate
            if prop.use_prefix:
                prefix = re.sub("_props$", "", name) + "_"
            else:
                prefix = ""

            for subpropname in delegate.class_properties(withbases=False):
                fullpropname = prefix + subpropname
                subprop = delegate.lookup(subpropname)
                if isinstance(subprop, Property):
                    # If it's an actual instance, then we need to make a copy
                    # so two properties don't write to the same hidden variable
                    # inside the instance.
                    subprop = copy(subprop)
                if "%s" in prop.help:
                    doc = prop.help % subpropname.replace('_', ' ')
                else:
                    doc = prop.help
                try:
                    includes[fullpropname] = subprop(help=doc)
                except TypeError:
                    includes[fullpropname] = subprop
                    subprop.__doc__ = doc
            # Remove the name of the Include attribute itself
            removes.add(name)

        # Update the class dictionary, taking care not to overwrite values
        # from the delegates that the subclass may have explicitly defined
        for key, val in includes.items():
            if key not in class_dict:
                class_dict[key] = val
        for tmp in removes:
            del class_dict[tmp]

        dataspecs = {}
        units_to_add = {}
        for name, prop in class_dict.items():
            if isinstance(prop, Property):
                prop.name = name
                if prop.has_ref:
                    names_with_refs.add(name)
                elif isinstance(prop, ContainerProperty):
                    container_names.add(name)
                names.add(name)
                if isinstance(prop, DataSpec):
                    dataspecs[name] = prop
                    if hasattr(prop, '_units_type'):
                        units_to_add[name+"_units"] = prop._units_type

            elif isinstance(prop, type) and issubclass(prop, Property):
                # Support the user adding a property without using parens,
                # i.e. using just the Property subclass instead of an
                # instance of the subclass
                newprop = prop.autocreate(name=name)
                class_dict[name] = newprop
                newprop.name = name
                names.add(name)

                # Process dataspecs
                if issubclass(prop, DataSpec):
                    dataspecs[name] = newprop

        for name, prop in units_to_add.items():
            prop.name = name
            names.add(name)
            class_dict[name] = prop

        class_dict["__properties__"] = names
        class_dict["__properties_with_refs__"] = names_with_refs
        class_dict["__container_props__"] = container_names
        if dataspecs:
            class_dict["_dataspecs"] = dataspecs
        return type.__new__(cls, class_name, bases, class_dict)

def accumulate_from_subclasses(cls, propname):
    s = set()
    for c in inspect.getmro(cls):
        if issubclass(c, HasProps):
            s.update(getattr(c, propname))
    return s

@add_metaclass(MetaHasProps)
class HasProps(object):

    def __init__(self, **properties):
        super(HasProps, self).__init__()
        self._changed_vars = set()

        for name, value in properties.items():
            setattr(self, name, value)

    def __setattr__(self, name, value):
        props = sorted(self.properties())

        if name.startswith("_") or name in props:
            super(HasProps, self).__setattr__(name, value)
        else:
            matches, text = difflib.get_close_matches(name.lower(), props), "similar"

            if not matches:
                matches, text = props, "possible"

            raise AttributeError("unexpected attribute '%s' to %s, %s attributes are %s" %
                (name, self.__class__.__name__, text, nice_join(matches)))

    def clone(self):
        """ Returns a duplicate of this object with all its properties
        set appropriately.  Values which are containers are shallow-copied.
        """
        return self.__class__(**self.changed_properties_with_values())

    @classmethod
    def lookup(cls, name):
        return getattr(cls, name)

    @classmethod
    def properties_with_refs(cls):
        """ Returns a set of the names of this object's properties that
        have references. We traverse the class hierarchy and
        pull together the full list of properties.
        """
        if not hasattr(cls, "__cached_allprops_with_refs"):
            s = accumulate_from_subclasses(cls, "__properties_with_refs__")
            cls.__cached_allprops_with_refs = s
        return cls.__cached_allprops_with_refs

    @classmethod
    def properties_containers(cls):
        """ Returns a list of properties that are containers
        """
        if not hasattr(cls, "__cached_allprops_containers"):
            s = accumulate_from_subclasses(cls, "__container_props__")
            cls.__cached_allprops_containers = s
        return cls.__cached_allprops_containers

    @classmethod
    def properties(cls):
        """ Returns a set of the names of this object's properties. We
        traverse the class hierarchy and pull together the full
        list of properties.
        """
        if not hasattr(cls, "__cached_allprops"):
            s = cls.class_properties()
            cls.__cached_allprops = s
        return cls.__cached_allprops

    @classmethod
    def dataspecs(cls):
        """ Returns a set of the names of this object's dataspecs (and
        dataspec subclasses).  Traverses the class hierarchy.
        """
        if not hasattr(cls, "__cached_dataspecs"):
            dataspecs = set()
            for c in reversed(inspect.getmro(cls)):
                if hasattr(c, "_dataspecs"):
                    dataspecs.update(c._dataspecs.keys())
            cls.__cached_dataspecs = dataspecs
        return cls.__cached_dataspecs

    @classmethod
    def dataspecs_with_refs(cls):
        dataspecs = {}
        for c in reversed(inspect.getmro(cls)):
            if hasattr(c, "_dataspecs"):
                dataspecs.update(c._dataspecs)
        return dataspecs

    def changed_vars(self):
        """ Returns which variables changed since the creation of the object,
        or the last called to reset_changed_vars().
        """
        return set.union(self._changed_vars, self.properties_with_refs(),
                         self.properties_containers())

    def reset_changed_vars(self):
        self._changed_vars = set()

    def properties_with_values(self):
        return dict([ (attr, getattr(self, attr)) for attr in self.properties() ])

    def changed_properties(self):
        return self.changed_vars()

    def changed_properties_with_values(self):
        return dict([ (attr, getattr(self, attr)) for attr in self.changed_properties() ])

    @classmethod
    def class_properties(cls, withbases=True):
        if withbases:
            return accumulate_from_subclasses(cls, "__properties__")
        else:
            return set(cls.__properties__)

    def set(self, **kwargs):
        """ Sets a number of properties at once """
        for kw in kwargs:
            setattr(self, kw, kwargs[kw])

    def pprint_props(self, indent=0):
        """ Prints the properties of this object, nicely formatted """
        for key, value in self.properties_with_values().items():
            print("%s%s: %r" % ("  "*indent, key, value))

class PrimitiveProperty(Property):
    """ A base class for simple property types. Subclasses should
    define a class attribute ``_underlying_type`` that is a tuple
    of acceptable type values for the property.

    """

    _underlying_type = None

    def validate(self, value):
        super(PrimitiveProperty, self).validate(value)

        if not (value is None or isinstance(value, self._underlying_type)):
            raise ValueError("expected a value of type %s, got %s of type %s" %
                (nice_join([ cls.__name__ for cls in self._underlying_type ]), value, type(value).__name__))

    def from_json(self, json, models=None):
        if json is None or isinstance(json, self._underlying_type):
            return json
        else:
            expected = nice_join([ cls.__name__ for cls in self._underlying_type ])
            raise DeserializationError("%s expected %s, got %s" % (self, expected, json))

class Bool(PrimitiveProperty):
    """ Boolean type property. """
    _underlying_type = (bool,)

class Int(PrimitiveProperty):
    """ Signed integer type property. """
    _underlying_type = bokeh_integer_types

class Float(PrimitiveProperty):
    """ Floating point type property. """
    _underlying_type = (float, ) + bokeh_integer_types

class Complex(PrimitiveProperty):
    """ Complex floating point type property. """
    _underlying_type = (complex, float) + bokeh_integer_types

class String(PrimitiveProperty):
    """ String type property. """
    _underlying_type = string_types

class Regex(String):
    """ Regex type property validates that text values match the
    given regular expression.
    """
    def __init__(self, regex, default=None, help=None):
        self.regex = re.compile(regex)
        super(Regex, self).__init__(default=default, help=help)

    def validate(self, value):
        super(Regex, self).validate(value)

        if not (value is None or self.regex.match(value) is not None):
            raise ValueError("expected a string matching %r pattern, got %r" % (self.regex.pattern, value))

    def __str__(self):
        return "%s(%r)" % (self.__class__.__name__, self.regex.pattern)

class JSON(String):
    """ JSON type property validates that text values are valid JSON.

    ..  note::
        The string is transmitted and received by BokehJS as a *string*
        containing JSON content. i.e., you must use ``JSON.parse`` to unpack
        the value into a JavaScript hash.

    """
    def validate(self, value):
        super(JSON, self).validate(value)

        if value is None: return

        try:
            import json
            json.loads(value)
        except ValueError:
            raise ValueError("expected JSON text, got %r" % value)

class ParameterizedProperty(Property):
    """ Base class for Properties that have type parameters, e.g.
    ``List(String)``.

    """

    @staticmethod
    def _validate_type_param(type_param):
        if isinstance(type_param, type):
            if issubclass(type_param, Property):
                return type_param()
            else:
                type_param = type_param.__name__
        elif isinstance(type_param, Property):
            return type_param

        raise ValueError("expected a property as type parameter, got %s" % type_param)

    @property
    def type_params(self):
        raise NotImplementedError("abstract method")

    @property
    def has_ref(self):
        return any(type_param.has_ref for type_param in self.type_params)

class ContainerProperty(ParameterizedProperty):
    """ Base class for Container-like type properties. """
    pass

class Seq(ContainerProperty):
    """ Sequence (list, tuple) type property.

    """

    def _is_seq(self, value):
        return isinstance(value, collections.Container) and not isinstance(value, collections.Mapping)

    def _new_instance(self, value):
        return value

    def __init__(self, item_type, default=None, help=None):
        self.item_type = self._validate_type_param(item_type)
        super(Seq, self).__init__(default=default, help=help)

    @property
    def type_params(self):
        return [self.item_type]

    def validate(self, value):
        super(Seq, self).validate(value)

        if value is not None:
            if not (self._is_seq(value) and all(self.item_type.is_valid(item) for item in value)):
                raise ValueError("expected an element of %s, got %r" % (self, value))

    def __str__(self):
        return "%s(%s)" % (self.__class__.__name__, self.item_type)

    def from_json(self, json, models=None):
        if json is None:
            return None
        elif isinstance(json, list):
            return self._new_instance([ self.item_type.from_json(item, models) for item in json ])
        else:
            raise DeserializationError("%s expected a list or None, got %s" % (self, json))

class List(Seq):
    """ Python list type property.

    """

    def __init__(self, item_type, default=[], help=None):
        # todo: refactor to not use mutable objects as default values.
        # Left in place for now because we want to allow None to express
        # opional values. Also in Dict.
        super(List, self).__init__(item_type, default=default, help=help)

    def _is_seq(self, value):
        return isinstance(value, list)

class Array(Seq):
    """ NumPy array type property.

    """

    def _is_seq(self, value):
        import numpy as np
        return isinstance(value, np.ndarray)

    def _new_instance(self, value):
        return np.array(value)

class Dict(ContainerProperty):
    """ Python dict type property.

    If a default value is passed in, then a shallow copy of it will be
    used for each new use of this property.

    """

    def __init__(self, keys_type, values_type, default={}, help=None):
        self.keys_type = self._validate_type_param(keys_type)
        self.values_type = self._validate_type_param(values_type)
        super(Dict, self).__init__(default=default, help=help)

    @property
    def type_params(self):
        return [self.keys_type, self.values_type]

    def validate(self, value):
        super(Dict, self).validate(value)

        if value is not None:
            if not (isinstance(value, dict) and \
                    all(self.keys_type.is_valid(key) and self.values_type.is_valid(val) for key, val in iteritems(value))):
                raise ValueError("expected an element of %s, got %r" % (self, value))

    def __str__(self):
        return "%s(%s, %s)" % (self.__class__.__name__, self.keys_type, self.values_type)

    def from_json(self, json, models=None):
        if json is None:
            return None
        elif isinstance(json, dict):
            return { self.keys_type.from_json(key, models): self.values_type.from_json(value, models) for key, value in iteritems(json) }
        else:
            raise DeserializationError("%s expected a dict or None, got %s" % (self, json))

class Tuple(ContainerProperty):
    """ Tuple type property. """
    def __init__(self, tp1, tp2, *type_params, **kwargs):
        self._type_params = list(map(self._validate_type_param, (tp1, tp2) + type_params))
        super(Tuple, self).__init__(default=kwargs.get("default"), help=kwargs.get("help"))

    @property
    def type_params(self):
        return self._type_params

    def validate(self, value):
        super(Tuple, self).validate(value)

        if value is not None:
            if not (isinstance(value, (tuple, list)) and len(self.type_params) == len(value) and \
                    all(type_param.is_valid(item) for type_param, item in zip(self.type_params, value))):
                raise ValueError("expected an element of %s, got %r" % (self, value))

    def __str__(self):
        return "%s(%s)" % (self.__class__.__name__, ", ".join(map(str, self.type_params)))

    def from_json(self, json, models=None):
        if json is None:
            return None
        elif isinstance(json, list):
            return tuple(type_param.from_json(item, models) for type_param, item in zip(self.type_params, json))
        else:
            raise DeserializationError("%s expected a list or None, got %s" % (self, json))

class Instance(Property):
    """ Instance type property, for references to other Models in the object
    graph.

    """
    def __init__(self, instance_type, default=None, help=None):
        if not isinstance(instance_type, (type,) + string_types):
            raise ValueError("expected a type or string, got %s" % instance_type)

        if isinstance(instance_type, type) and not issubclass(instance_type, HasProps):
            raise ValueError("expected a subclass of HasProps, got %s" % instance_type)

        self._instance_type = instance_type

        super(Instance, self).__init__(default=default, help=help)

    @property
    def instance_type(self):
        if isinstance(self._instance_type, str):
            module, name = self._instance_type.rsplit(".", 1)
            self._instance_type = getattr(import_module(module, "bokeh"), name)

        return self._instance_type

    @property
    def has_ref(self):
        return True

    def validate(self, value):
        super(Instance, self).validate(value)

        if value is not None:
            if not isinstance(value, self.instance_type):
                raise ValueError("expected an instance of type %s, got %s of type %s" %
                    (self.instance_type.__name__, value, type(value).__name__))

    def __str__(self):
        return "%s(%s)" % (self.__class__.__name__, self.instance_type.__name__)

    def from_json(self, json, models=None):
        if json is None:
            return None
        elif isinstance(json, dict):
            from .plot_object import PlotObject
            if issubclass(self.instance_type, PlotObject):
                if models is None:
                    raise DeserializationError("%s can't deserialize without models" % self)
                else:
                    model = models.get(json["id"])

                    if model is not None:
                        return model
                    else:
                        raise DeserializationError("%s failed to deserilize reference to %s" % (self, json))
            else:
                attrs = {}

                for name, value in iteritems(json):
                    prop = self.instance_type.lookup(name)
                    attrs[name] = prop.from_json(value, models)

                # XXX: this doesn't work when Instance(Superclass) := Subclass()
                # Serialization dict must carry type information to resolve this.
                return self.instance_type(**attrs)
        else:
            raise DeserializationError("%s expected a dict or None, got %s" % (self, json))

class This(Property):
    """ A reference to an instance of the class being defined. """
    pass

# Fake types, ABCs
class Any(Property):
    """ Any type property accepts any values. """
    pass

class Function(Property):
    """ Function type property. """
    pass

class Event(Property):
    """ Event type property. """
    pass

class Interval(ParameterizedProperty):
    ''' Range type property ensures values are contained inside a given interval. '''
    def __init__(self, interval_type, start, end, default=None, help=None):
        self.interval_type = self._validate_type_param(interval_type)
        self.interval_type.validate(start)
        self.interval_type.validate(end)
        self.start = start
        self.end = end
        super(Interval, self).__init__(default=default, help=help)

    @property
    def type_params(self):
        return [self.interval_type]

    def validate(self, value):
        super(Interval, self).validate(value)

        if not (value is None or self.interval_type.is_valid(value) and value >= self.start and value <= self.end):
            raise ValueError("expected a value of type %s in range [%s, %s], got %r" % (self.interval_type, self.start, self.end, value))

    def __str__(self):
        return "%s(%s, %r, %r)" % (self.__class__.__name__, self.interval_type, self.start, self.end)

class Byte(Interval):
    ''' Byte type property. '''
    def __init__(self, default=0, help=None):
        super(Byte, self).__init__(Int, 0, 255, default=default, help=help)

class Either(ParameterizedProperty):
    """ Takes a list of valid properties and validates against them in succession. """

    def __init__(self, tp1, tp2, *type_params, **kwargs):
        self._type_params = list(map(self._validate_type_param, (tp1, tp2) + type_params))
        default = kwargs.get("default", self._type_params[0].default)
        help = kwargs.get("help")
        super(Either, self).__init__(default=default, help=help)

    @property
    def type_params(self):
        return self._type_params

    def validate(self, value):
        super(Either, self).validate(value)

        if not (value is None or any(param.is_valid(value) for param in self.type_params)):
            raise ValueError("expected an element of either %s, got %r" % (nice_join(self.type_params), value))

    def transform(self, value):
        for param in self.type_params:
            try:
                return param.transform(value)
            except ValueError:
                pass

        raise ValueError("Could not transform %r" % value)

    def from_json(self, json, models=None):
        for tp in self.type_params:
            try:
                return tp.from_json(json, models)
            except DeserializationError:
                pass
        else:
            raise DeserializationError("%s couldn't deserialize %s" % (self, json))

    def __str__(self):
        return "%s(%s)" % (self.__class__.__name__, ", ".join(map(str, self.type_params)))

    def __or__(self, other):
        return self.__class__(*(self.type_params + [other]), default=self._default, help=self.help)

class Enum(Property):
    """ An Enum with a list of allowed values. The first value in the list is
    the default value, unless a default is provided with the "default" keyword
    argument.
    """
    def __init__(self, enum, *values, **kwargs):
        if not (not values and isinstance(enum, enums.Enumeration)):
            enum = enums.enumeration(enum, *values)

        self.allowed_values = enum._values

        default = kwargs.get("default", enum._default)
        help = kwargs.get("help")
        super(Enum, self).__init__(default=default, help=help)

    def validate(self, value):
        super(Enum, self).validate(value)

        if not (value is None or value in self.allowed_values):
            raise ValueError("invalid value for %s: %r; allowed values are %s" % (self.name, value, nice_join(self.allowed_values)))

    def __str__(self):
        return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, self.allowed_values)))

class Auto(Enum):

    def __init__(self):
        super(Auto, self).__init__("auto")

    def __str__(self):
        return self.__class__.__name__

# Properties useful for defining visual attributes
class Color(Either):
    """ Accepts color definition in a variety of ways, and produces an
    appropriate serialization of its value for whatever backend.

    For colors, because we support named colors and hex values prefaced
    with a "#", when we are handed a string value, there is a little
    interpretation: if the value is one of the 147 SVG named colors or
    it starts with a "#", then it is interpreted as a value.

    If a 3-tuple is provided, then it is treated as an RGB (0..255).
    If a 4-tuple is provided, then it is treated as an RGBa (0..255), with
    alpha as a float between 0 and 1.  (This follows the HTML5 Canvas API.)
    """

    def __init__(self, default=None, help=None):
        types = (Enum(enums.NamedColor),
                 Regex("^#[0-9a-fA-F]{6}$"),
                 Tuple(Byte, Byte, Byte),
                 Tuple(Byte, Byte, Byte, Percent))
        super(Color, self).__init__(*types, default=default, help=help)

    def __str__(self):
        return self.__class__.__name__

class Align(Property):
    pass

class DashPattern(Either):
    """ Dash type property.

    Express patterns that describe line dashes.  ``DashPattern`` values
    can be specified in a variety of ways:

    * An enum: "solid", "dashed", "dotted", "dotdash", "dashdot"
    * a tuple or list of integers in the `HTML5 Canvas dash specification style`_.
      Note that if the list of integers has an odd number of elements, then
      it is duplicated, and that duplicated list becomes the new dash list.

    To indicate that dashing is turned off (solid lines), specify the empty
    list [].

    .. _HTML5 Canvas dash specification style: http://www.w3.org/html/wg/drafts/2dcontext/html5_canvas/#dash-list

    """

    _dash_patterns = {
        "solid": [],
        "dashed": [6],
        "dotted": [2,4],
        "dotdash": [2,4,6,4],
        "dashdot": [6,4,2,4],
    }

    def __init__(self, default=[], help=None):
        types = Enum(enums.DashPattern), Regex(r"^(\d+(\s+\d+)*)?$"), Seq(Int)
        super(DashPattern, self).__init__(*types, default=default, help=help)

    def transform(self, value):
        value = super(DashPattern, self).transform(value)

        if isinstance(value, string_types):
            try:
                return self._dash_patterns[value]
            except KeyError:
                return [int(x) for x in  value.split()]
        else:
            return value

    def __str__(self):
        return self.__class__.__name__

class Size(Float):
    """ Size type property.

    .. note::
        ``Size`` is equivalent to an unsigned int.

    """
    def validate(self, value):
        super(Size, self).validate(value)

        if not (value is None or 0.0 <= value):
            raise ValueError("expected a non-negative number, got %r" % value)

class Percent(Float):
    """ Percentage type property.

    Percents are useful for specifying alphas and coverage and extents; more
    semantically meaningful than Float(0..1).

    """
    def validate(self, value):
        super(Percent, self).validate(value)

        if not (value is None or 0.0 <= value <= 1.0):
            raise ValueError("expected a value in range [0, 1], got %r" % value)

class Angle(Float):
    """ Angle type property. """
    pass

class Date(Property):
    """ Date (not datetime) type property.

    """
    def __init__(self, default=datetime.date.today(), help=None):
        super(Date, self).__init__(default=default, help=help)

    def validate(self, value):
        super(Date, self).validate(value)

        if not (value is None or isinstance(value, (datetime.date,) + string_types + (float,) + bokeh_integer_types)):
            raise ValueError("expected a date, string or timestamp, got %r" % value)

    def transform(self, value):
        value = super(Date, self).transform(value)

        if isinstance(value, (float,) + bokeh_integer_types):
            try:
                value = datetime.date.fromtimestamp(value)
            except ValueError:
                value = datetime.date.fromtimestamp(value/1000)
        elif isinstance(value, string_types):
            value = dateutil.parser.parse(value).date()

        return value

class Datetime(Property):
    """ Datetime type property.

    """

    def __init__(self, default=datetime.date.today(), help=None):
        super(Datetime, self).__init__(default=default, help=help)

    def validate(self, value):
        super(Datetime, self).validate(value)

        if (isinstance(value, (datetime.datetime, datetime.date, np.datetime64))):
            return
        try:
            import pandas
            if isinstance(value, (pandas.Timestamp)):
                return
        except ImportError:
            pass

        raise ValueError("Expected a datetime instance, got %r" % value)

    def transform(self, value):
        value = super(Datetime, self).transform(value)
        return value
        # Handled by serialization in protocol.py for now

class RelativeDelta(Dict):
    """ RelativeDelta type property for time deltas.

    """

    def __init__(self, default={}, help=None):
        keys = Enum("years", "months", "days", "hours", "minutes", "seconds", "microseconds")
        values = Int
        super(RelativeDelta, self).__init__(keys, values, default=default, help=help)

    def __str__(self):
        return self.__class__.__name__

class DataSpec(Either):
    def __init__(self, typ, default, help=None):
        super(DataSpec, self).__init__(String, Dict(String, Either(String, typ)), typ, default=default, help=help)
        self._type = self._validate_type_param(typ)

    def to_dict(self, obj):
        val = getattr(obj, self._name, self.default)

        # Check for None value
        if val is None:
            return dict(value=None)

        # Check for spec type value
        try:
            self._type.validate(val)
            return dict(value=val)
        except ValueError:
            pass

        # Check for data source field name
        if isinstance(val, string_types):
            return dict(field=val)

        # Must be dict, return as-is
        return val

    def __str__(self):
        val = getattr(self, self._name, self.default)
        return "%s(%r)" % (self.__class__.__name__, val)

class NumberSpec(DataSpec):
    def __init__(self, default, help=None):
        super(NumberSpec, self).__init__(Float, default=default, help=help)

class StringSpec(DataSpec):
    def __init__(self, default, help=None):
        super(StringSpec, self).__init__(List(String), default=default, help=help)

    def __set__(self, obj, value):
        if isinstance(value, list):
            if len(value) != 1:
                raise TypeError("StringSpec convenience list values must have length 1")
            value = dict(value=value[0])
        super(StringSpec, self).__set__(obj, value)

class FontSizeSpec(DataSpec):
    def __init__(self, default, help=None):
        super(FontSizeSpec, self).__init__(List(String), default=default, help=help)

    def __set__(self, obj, value):
        if isinstance(value, string_types):
            warn('Setting a fixed font size value as a string %r is deprecated, '
                 'set with value(%r) or [%r] instead' % (value, value, value),
                 DeprecationWarning, stacklevel=2)
            if len(value) > 0 and value[0].isdigit():
                value = dict(value=value)
        super(FontSizeSpec, self).__set__(obj, value)

class UnitsSpec(NumberSpec):
    def __init__(self, default, units_type, units_default, help=None):
        super(UnitsSpec, self).__init__(default=default, help=help)
        self._units_type = self._validate_type_param(units_type)
        self._units_type.validate(units_default)
        self._units_type._default = units_default

    def to_dict(self, obj):
        d = super(UnitsSpec, self).to_dict(obj)
        d["units"] = getattr(obj, self.name+"_units")
        return d

    def __set__(self, obj, value):
        if isinstance(value, dict):
            units = value.pop("units", None)
            if units: setattr(obj, self.name+"_units", units)
        super(UnitsSpec, self).__set__(obj, value)

    def __str__(self):
        val = getattr(self, self._name, self.default)
        return "%s(%r, units_default=%r)" % (self.__class__.__name__, val, self._units_type._default)

class AngleSpec(UnitsSpec):
    def __init__(self, default, units_default="rad", help=None):
        super(AngleSpec, self).__init__(default=default, units_type=Enum(enums.AngleUnits), units_default=units_default, help=help)

class DistanceSpec(UnitsSpec):
    def __init__(self, default, units_default="data", help=None):
        super(DistanceSpec, self).__init__(default=default, units_type=Enum(enums.SpatialUnits), units_default=units_default, help=help)

    def __set__(self, obj, value):
        try:
            if value < 0:
                raise ValueError("Distances must be non-negative")
        except TypeError:
            pass
        super(DistanceSpec, self).__set__(obj, value)

class ScreenDistanceSpec(NumberSpec):
    def to_dict(self, obj):
        d = super(ScreenDistanceSpec, self).to_dict(obj)
        d["units"] = "screen"
        return d

    def __set__(self, obj, value):
        try:
            if value < 0:
                raise ValueError("Distances must be non-negative")
        except TypeError:
            pass
        super(ScreenDistanceSpec, self).__set__(obj, value)

class DataDistanceSpec(NumberSpec):
    def to_dict(self, obj):
        d = super(ScreenDistanceSpec, self).to_dict(obj)
        d["units"] = "data"
        return d

    def __set__(self, obj, value):
        try:
            if value < 0:
                raise ValueError("Distances must be non-negative")
        except TypeError:
            pass
        super(DataDistanceSpec, self).__set__(obj, value)

class ColorSpec(DataSpec):
    def __init__(self, default, help=None):
        super(ColorSpec, self).__init__(Color, default=default, help=help)

    @classmethod
    def isconst(cls, arg):
        """ Returns True if the argument is a literal color.  Check for a
        well-formed hexadecimal color value.
        """
        return isinstance(arg, string_types) and \
               ((len(arg) == 7 and arg[0] == "#") or arg in enums.NamedColor._values)

    @classmethod
    def is_color_tuple(cls, val):
        return isinstance(val, tuple) and len(val) in (3, 4)

    @classmethod
    def format_tuple(cls, colortuple):
        if len(colortuple) == 3:
            return "rgb%r" % (colortuple,)
        else:
            return "rgba%r" % (colortuple,)

    def to_dict(self, obj):
        val = getattr(obj, self._name, self.default)

        if val is None:
            return dict(value=None)

        # Check for hexadecimal or named color
        if self.isconst(val):
            return dict(value=val)

        # Check for RGB or RGBa tuple
        if isinstance(val, tuple):
            return dict(value=self.format_tuple(val))

        # Check for data source field name
        if isinstance(val, string_types):
            return dict(field=val)

        # Must be dict, return as-is
        return val

    def validate(self, value):
        try:
            return super(ColorSpec, self).validate(value)
        except ValueError as e:
            # Check for tuple input if not yet a valid input type
            if self.is_color_tuple(value):
                return True
            else:
                raise e

    def transform(self, value):

        # Make sure that any tuple has either three integers, or three integers and one float
        if isinstance(value, tuple):
            value = tuple(int(v) if i < 3 else v for i, v in enumerate(value))

        return value


# vim: set encoding=utf-8

#  Copyright (c) 2016 Intel Corporation
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

""" Test frame.box_cox() and frame.reverse_box_cox()"""

import unittest
from sparktkregtests.lib import sparktk_test

class BoxCoxTest(sparktk_test.SparkTKTestCase):

    def setUp(self):
        """Build test frame"""
        super(BoxCoxTest, self).setUp()

        dataset =\
            [[5.8813080107727425], [8.9771372790941797], [8.9153072947470804],
            [8.1583747730768401], [0.35889585616853292]]
        schema = [("y", float)]

        self.frame = self.context.frame.create(dataset, schema=schema)

    def test_wt_default(self):
        """ Test behaviour for default params, lambda = 0 """
        self.frame.box_cox("y")

        actual = self.frame.to_pandas()["y_lambda_0.0"].tolist()
        expected =\
            [1.7717791879837133, 2.1946810429706676,
            2.1877697201262163, 2.0990449791729704, -1.0247230268174008]
        self.assertItemsEqual(actual, expected)

    def test_lambda(self):
        """ Test wt for lambda = 0.3 """
        self.frame.box_cox("y", 0.3)

        actual = self.frame.to_pandas()["y_lambda_0.3"].tolist()
        expected =\
            [2.3384668540844573, 3.1056915770236082,
            3.0923547540771801, 2.9235756971904037, -0.88218677941017198]
        self.assertItemsEqual(actual, expected)

    def test_reverse_default(self):
        """ Test reverse transform for default lambda = 0 """
        self.frame.box_cox("y")
        self.frame.reverse_box_cox("y_lambda_0.0",
            reverse_box_cox_column_name="reverse")

        actual = self.frame.to_pandas()["reverse"].tolist()
        expected =\
            [5.8813080107727425, 8.9771372790941815,
            8.9153072947470804, 8.1583747730768401, 0.35889585616853298]

        self.assertItemsEqual(actual, expected)

    def test_reverse_lambda(self):
        """ Test reverse transform for lambda = 0.3 """
        self.frame.box_cox("y", 0.3)
        self.frame.reverse_box_cox("y_lambda_0.3", 0.3,
            reverse_box_cox_column_name="reverse")

        actual = self.frame.to_pandas()["reverse"].tolist()
        expected =\
            [5.8813080107727442, 8.9771372790941797,
            8.9153072947470822, 8.1583747730768419,
            0.35889585616853298]

        self.assertItemsEqual(actual, expected)

    @unittest.skip("req not clear")
    def test_lambda_negative(self):
        """ Test box cox for lambda -1 """
        self.frame.box_cox("y", -1)

        actual = self.frame.to_pandas()["y_lambda_-1.0"].tolist()
        expected =\
            [0.82996979614597488, 0.88860591423406388,
            0.88783336715839256, 0.87742656744575354,
            -1.7863236167608822]

        self.assertItemsEqual(actual, expected)

    def test_existing_boxcox_column(self):
        """ Test behavior for existing boxcox column """
        self.frame.box_cox("y", 0.3)

        with self.assertRaisesRegexp(
                Exception, "duplicate column name"):
            self.frame.box_cox("y", 0.3)

    def test_existing_reverse_column(self):
        """ Test behavior for existing reverse boxcox column """
        self.frame.reverse_box_cox("y", 0.3)

        with self.assertRaisesRegexp(
                Exception, "duplicate column name"):
            self.frame.reverse_box_cox("y", 0.3)

    @unittest.skip("Req not clear")
    def test_negative_col_positive_lambda(self):
        """Test behaviour for negative input column and positive lambda"""
        frame = self.context.frame.create([[-1], [-2], [1]], [("y", float)])
        frame.box_cox("y", 1)

        actual = frame.to_pandas()["y_lambda_1.0"].tolist()
        expected = [-2.0, -3.0, 0]

        self.assertItemsEqual(actual, expected)

    @unittest.skip("Req not clear")
    def test_negative_col_frational_lambda(self):
        """Test behaviour for negative input column and negative lambda"""
        frame = self.context.frame.create([[-1], [-2], [1]], [("y", float)])

        with self.assertRaises(Exception):
            frame.box_cox("y", 0.1)

    @unittest.skip("Req not clear")
    def test_negative_col_zero_lambda(self):
        """Test behaviour for negative input column and positive lambda"""
        frame = self.context.frame.create([[-1], [-2], [1]], [("y", float)])

        with self.assertRaises(Exception):
            frame.box_cox("y")

if __name__ == "__main__":
    unittest.main()


"""
=======================================
Visualizing the stock market structure
=======================================

This example employs several unsupervised learning techniques to extract
the stock market structure from variations in historical quotes.

The quantity that we use is the daily variation in quote price: quotes
that are linked tend to cofluctuate during a day.

.. _stock_market:

Learning a graph structure
--------------------------

We use sparse inverse covariance estimation to find which quotes are
correlated conditionally on the others. Specifically, sparse inverse
covariance gives us a graph, that is a list of connection. For each
symbol, the symbols that it is connected too are those useful to explain
its fluctuations.

Clustering
----------

We use clustering to group together quotes that behave similarly. Here,
amongst the :ref:`various clustering techniques <clustering>` available
in the scikit-learn, we use :ref:`affinity_propagation` as it does
not enforce equal-size clusters, and it can choose automatically the
number of clusters from the data.

Note that this gives us a different indication than the graph, as the
graph reflects conditional relations between variables, while the
clustering reflects marginal properties: variables clustered together can
be considered as having a similar impact at the level of the full stock
market.

Embedding in 2D space
---------------------

For visualization purposes, we need to lay out the different symbols on a
2D canvas. For this we use :ref:`manifold` techniques to retrieve 2D
embedding.

Visualization
-------------

The output of the 3 models are combined in a 2D graph where nodes
represents the stocks and edges the:

- cluster labels are used to define the color of the nodes
- the sparse covariance model is used to display the strength of the edges
- the 2D embedding is used to position the nodes in the plan

This example has a fair amount of visualization-related code, as
visualization is crucial here to display the graph. One of the challenge
is to position the labels minimizing overlap. For this we use an
heuristic based on the direction of the nearest neighbor along each
axis.
"""
print(__doc__)

# Author: Gael Varoquaux gael.varoquaux@normalesup.org
# License: BSD 3 clause

import datetime

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import finance
from matplotlib.collections import LineCollection

from sklearn import cluster, covariance, manifold

###############################################################################
# Retrieve the data from Internet

# Choose a time period reasonnably calm (not too long ago so that we get
# high-tech firms, and before the 2008 crash)
d1 = datetime.datetime(2003, 1, 1)
d2 = datetime.datetime(2008, 1, 1)

# kraft symbol has now changed from KFT to MDLZ in yahoo
symbol_dict = {
    'TOT': 'Total',
    'XOM': 'Exxon',
    'CVX': 'Chevron',
    'COP': 'ConocoPhillips',
    'VLO': 'Valero Energy',
    'MSFT': 'Microsoft',
    'IBM': 'IBM',
    'TWX': 'Time Warner',
    'CMCSA': 'Comcast',
    'CVC': 'Cablevision',
    'YHOO': 'Yahoo',
    'DELL': 'Dell',
    'HPQ': 'HP',
    'AMZN': 'Amazon',
    'TM': 'Toyota',
    'CAJ': 'Canon',
    'MTU': 'Mitsubishi',
    'SNE': 'Sony',
    'F': 'Ford',
    'HMC': 'Honda',
    'NAV': 'Navistar',
    'NOC': 'Northrop Grumman',
    'BA': 'Boeing',
    'KO': 'Coca Cola',
    'MMM': '3M',
    'MCD': 'Mc Donalds',
    'PEP': 'Pepsi',
    'MDLZ': 'Kraft Foods',
    'K': 'Kellogg',
    'UN': 'Unilever',
    'MAR': 'Marriott',
    'PG': 'Procter Gamble',
    'CL': 'Colgate-Palmolive',
    'GE': 'General Electrics',
    'WFC': 'Wells Fargo',
    'JPM': 'JPMorgan Chase',
    'AIG': 'AIG',
    'AXP': 'American express',
    'BAC': 'Bank of America',
    'GS': 'Goldman Sachs',
    'AAPL': 'Apple',
    'SAP': 'SAP',
    'CSCO': 'Cisco',
    'TXN': 'Texas instruments',
    'XRX': 'Xerox',
    'LMT': 'Lookheed Martin',
    'WMT': 'Wal-Mart',
    'WBA': 'Walgreen',
    'HD': 'Home Depot',
    'GSK': 'GlaxoSmithKline',
    'PFE': 'Pfizer',
    'SNY': 'Sanofi-Aventis',
    'NVS': 'Novartis',
    'KMB': 'Kimberly-Clark',
    'R': 'Ryder',
    'GD': 'General Dynamics',
    'RTN': 'Raytheon',
    'CVS': 'CVS',
    'CAT': 'Caterpillar',
    'DD': 'DuPont de Nemours'}

symbols, names = np.array(list(symbol_dict.items())).T

quotes = [finance.quotes_historical_yahoo(symbol, d1, d2, asobject=True)
          for symbol in symbols]

open = np.array([q.open for q in quotes]).astype(np.float)
close = np.array([q.close for q in quotes]).astype(np.float)

# The daily variations of the quotes are what carry most information
variation = close - open

###############################################################################
# Learn a graphical structure from the correlations
edge_model = covariance.GraphLassoCV()

# standardize the time series: using correlations rather than covariance
# is more efficient for structure recovery
X = variation.copy().T
X /= X.std(axis=0)
edge_model.fit(X)

###############################################################################
# Cluster using affinity propagation

_, labels = cluster.affinity_propagation(edge_model.covariance_)
n_labels = labels.max()

for i in range(n_labels + 1):
    print('Cluster %i: %s' % ((i + 1), ', '.join(names[labels == i])))

###############################################################################
# Find a low-dimension embedding for visualization: find the best position of
# the nodes (the stocks) on a 2D plane

# We use a dense eigen_solver to achieve reproducibility (arpack is
# initiated with random vectors that we don't control). In addition, we
# use a large number of neighbors to capture the large-scale structure.
node_position_model = manifold.LocallyLinearEmbedding(
    n_components=2, eigen_solver='dense', n_neighbors=6)

embedding = node_position_model.fit_transform(X.T).T

###############################################################################
# Visualization
plt.figure(1, facecolor='w', figsize=(10, 8))
plt.clf()
ax = plt.axes([0., 0., 1., 1.])
plt.axis('off')

# Display a graph of the partial correlations
partial_correlations = edge_model.precision_.copy()
d = 1 / np.sqrt(np.diag(partial_correlations))
partial_correlations *= d
partial_correlations *= d[:, np.newaxis]
non_zero = (np.abs(np.triu(partial_correlations, k=1)) > 0.02)

# Plot the nodes using the coordinates of our embedding
plt.scatter(embedding[0], embedding[1], s=100 * d ** 2, c=labels,
            cmap=plt.cm.spectral)

# Plot the edges
start_idx, end_idx = np.where(non_zero)
#a sequence of (*line0*, *line1*, *line2*), where::
#            linen = (x0, y0), (x1, y1), ... (xm, ym)
segments = [[embedding[:, start], embedding[:, stop]]
            for start, stop in zip(start_idx, end_idx)]
values = np.abs(partial_correlations[non_zero])
lc = LineCollection(segments,
                    zorder=0, cmap=plt.cm.hot_r,
                    norm=plt.Normalize(0, .7 * values.max()))
lc.set_array(values)
lc.set_linewidths(15 * values)
ax.add_collection(lc)

# Add a label to each node. The challenge here is that we want to
# position the labels to avoid overlap with other labels
for index, (name, label, (x, y)) in enumerate(
        zip(names, labels, embedding.T)):

    dx = x - embedding[0]
    dx[index] = 1
    dy = y - embedding[1]
    dy[index] = 1
    this_dx = dx[np.argmin(np.abs(dy))]
    this_dy = dy[np.argmin(np.abs(dx))]
    if this_dx > 0:
        horizontalalignment = 'left'
        x = x + .002
    else:
        horizontalalignment = 'right'
        x = x - .002
    if this_dy > 0:
        verticalalignment = 'bottom'
        y = y + .002
    else:
        verticalalignment = 'top'
        y = y - .002
    plt.text(x, y, name, size=10,
             horizontalalignment=horizontalalignment,
             verticalalignment=verticalalignment,
             bbox=dict(facecolor='w',
                       edgecolor=plt.cm.spectral(label / float(n_labels)),
                       alpha=.6))

plt.xlim(embedding[0].min() - .15 * embedding[0].ptp(),
         embedding[0].max() + .10 * embedding[0].ptp(),)
plt.ylim(embedding[1].min() - .03 * embedding[1].ptp(),
         embedding[1].max() + .03 * embedding[1].ptp())

plt.show()


#!/usr/bin/env python
# coding: utf-8

# Minimal Contact Binary System
# ============================
#
# Setup
# -----------------------------

# Let's first make sure we have the latest version of PHOEBE 2.1 installed. (You can comment out this line if you don't use pip for your installation or don't want to update to the latest release).

# In[ ]:

get_ipython().system('pip install -I "phoebe>=2.1,<2.2"')

# As always, let's do imports and initialize a logger and a new bundle.  See [Building a System](../tutorials/building_a_system.html) for more details.

# In[1]:

get_ipython().run_line_magic('matplotlib', 'inline')

# In[2]:

import phoebe
from phoebe import u # units
import numpy as np
import matplotlib.pyplot as plt

logger = phoebe.logger()

# Here we'll initialize a default binary, but ask for it to be created as a contact system.

# In[3]:

b_cb = phoebe.default_binary(contact_binary=True)

# We'll compare this to the default detached binary

# In[4]:

b_detached = phoebe.default_binary()

# Hierarchy
# -------------

# Let's first look at the hierarchy of the default detached binary, and then compare that to the hierarchy of the overcontact system

# In[5]:

print b_detached.hierarchy

# In[6]:

print b_cb.hierarchy

# As you can see, the overcontact system has an additional "component" with method "envelope" and component label "contact_envelope".
#
# Next let's look at the parameters in the envelope and star components. You can see that most of parameters in the envelope class are constrained, while the equivalent radius of the primary is unconstrained. The value of primary equivalent radius constrains the potential and fillout factor of the envelope, as well as the equivalent radius of the secondary.

# In[7]:

print b_cb.filter(component='contact_envelope', kind='envelope', context='component')

# In[8]:

print b_cb.filter(component='primary', kind='star', context='component')

# In[9]:

b_cb['requiv@primary'] = 1.5

# In[10]:

b_cb['pot@contact_envelope@component']

# In[11]:

b_cb['fillout_factor@contact_envelope@component']

# In[12]:

b_cb['requiv@secondary@component']

# Now, of course, if we didn't originally know we wanted a contact binary and built the default detached system, we could still turn it into an contact binary just by changing the hierarchy.

# In[13]:

b_detached.add_component('envelope', component='contact_envelope')

# In[14]:

hier = phoebe.hierarchy.binaryorbit(b_detached['binary'], b_detached['primary'], b_detached['secondary'], b_detached['contact_envelope'])
print hier

# In[15]:

b_detached.set_hierarchy(hier)

# In[16]:

print b_detached.hierarchy

# However, since our system was detached, the system is not overflowing, and therefore doesn't pass system checks

# In[17]:

b_detached.run_checks()

# And because of this, the potential and requiv@secondary constraints cannot be computed

# In[18]:

b_detached['pot@component']

# In[19]:

b_detached['requiv@secondary@component']

# Likewise, we can make a contact system detached again simply by removing the envelope from the hierarchy.  The parameters themselves will still exist (unless you remove them), so you can always just change the hierarchy again to change back to an overcontact system.

# In[20]:

hier = phoebe.hierarchy.binaryorbit(b_detached['binary'], b_detached['primary'], b_detached['secondary'])
print hier

# In[21]:

b_detached.set_hierarchy(hier)

# In[22]:

print b_detached.hierarchy

# Although the constraints have been removed, PHOEBE has lost the original value of the secondary radius (because of the failed contact constraints), so we'll have to reset that here as well.

# In[23]:

b_detached['requiv@secondary'] = 1.0

# Adding Datasets
# ---------------------

# In[24]:

b_cb.add_dataset('mesh', times=[0], dataset='mesh01')

# In[25]:

b_cb.add_dataset('orb', times=np.linspace(0,1,201), dataset='orb01')

# In[26]:

b_cb.add_dataset('lc', times=np.linspace(0,1,21), dataset='lc01')

# In[27]:

b_cb.add_dataset('rv', times=np.linspace(0,1,21), dataset='rv01')

# For comparison, we'll do the same to our detached system

# In[28]:

b_detached.add_dataset('mesh', times=[0], dataset='mesh01')

# In[29]:

b_detached.add_dataset('orb', times=np.linspace(0,1,201), dataset='orb01')

# In[30]:

b_detached.add_dataset('lc', times=np.linspace(0,1,21), dataset='lc01')

# In[31]:

b_detached.add_dataset('rv', times=np.linspace(0,1,21), dataset='rv01')

# Running Compute
# --------------------

# In[32]:

b_cb.run_compute(irrad_method='none')

# In[33]:

b_detached.run_compute(irrad_method='none')

# Synthetics
# ------------------

# To ensure compatibility with computing synthetics in detached and semi-detached systems in Phoebe, the synthetic meshes for our overcontact system are attached to each component separetely, instead of the contact envelope.

# In[34]:

print b_cb['mesh01@model'].components

# In[35]:

print b_detached['mesh01@model'].components

# Plotting
# ---------------

# ### Meshes

# In[36]:

afig, mplfig = b_cb['mesh01@model'].plot(x='ws', show=True)

# In[37]:

afig, mplfig = b_detached['mesh01@model'].plot(x='ws', show=True)

# ### Orbits

# In[38]:

afig, mplfig = b_cb['orb01@model'].plot(x='ws',show=True)

# In[39]:

afig, mplfig = b_detached['orb01@model'].plot(x='ws',show=True)

# ### Light Curves

# In[40]:

afig, mplfig = b_cb['lc01@model'].plot(show=True)

# In[41]:

afig, mplfig = b_detached['lc01@model'].plot(show=True)

# ### RVs

# In[42]:

afig, mplfig = b_cb['rv01@model'].plot(show=True)

# In[43]:

afig, mplfig = b_detached['rv01@model'].plot(show=True)

# In[ ]:


"""
==============================================
Feature agglomeration vs. univariate selection
==============================================

This example compares 2 dimensionality reduction strategies:

- univariate feature selection with Anova

- feature agglomeration with Ward hierarchical clustering

Both methods are compared in a regression problem using
a BayesianRidge as supervised estimator.
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

print(__doc__)

import shutil
import tempfile

import numpy as np
import matplotlib.pyplot as plt
from scipy import linalg, ndimage

from sklearn.feature_extraction.image import grid_to_graph
from sklearn import feature_selection
from sklearn.cluster import FeatureAgglomeration
from sklearn.linear_model import BayesianRidge
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.externals.joblib import Memory
from sklearn.cross_validation import KFold

###############################################################################
# Generate data
n_samples = 200
size = 40  # image size
roi_size = 15
snr = 5.
np.random.seed(0)
mask = np.ones([size, size], dtype=np.bool)

coef = np.zeros((size, size))
coef[0:roi_size, 0:roi_size] = -1.
coef[-roi_size:, -roi_size:] = 1.

X = np.random.randn(n_samples, size ** 2)
for x in X:  # smooth data
    x[:] = ndimage.gaussian_filter(x.reshape(size, size), sigma=1.0).ravel()
X -= X.mean(axis=0)
X /= X.std(axis=0)

y = np.dot(X, coef.ravel())
noise = np.random.randn(y.shape[0])
noise_coef = (linalg.norm(y, 2) / np.exp(snr / 20.)) / linalg.norm(noise, 2)
y += noise_coef * noise  # add noise

###############################################################################
# Compute the coefs of a Bayesian Ridge with GridSearch
cv = KFold(len(y), 2)  # cross-validation generator for model selection
ridge = BayesianRidge()
cachedir = tempfile.mkdtemp()
mem = Memory(cachedir=cachedir, verbose=1)

# Ward agglomeration followed by BayesianRidge
connectivity = grid_to_graph(n_x=size, n_y=size)
ward = FeatureAgglomeration(n_clusters=10, connectivity=connectivity,
                            memory=mem)
clf = Pipeline([('ward', ward), ('ridge', ridge)])
# Select the optimal number of parcels with grid search
clf = GridSearchCV(clf, {'ward__n_clusters': [10, 20, 30]}, n_jobs=1, cv=cv)
clf.fit(X, y)  # set the best parameters
coef_ = clf.best_estimator_.steps[-1][1].coef_
coef_ = clf.best_estimator_.steps[0][1].inverse_transform(coef_)
coef_agglomeration_ = coef_.reshape(size, size)

# Anova univariate feature selection followed by BayesianRidge
f_regression = mem.cache(feature_selection.f_regression)  # caching function
anova = feature_selection.SelectPercentile(f_regression)
clf = Pipeline([('anova', anova), ('ridge', ridge)])
# Select the optimal percentage of features with grid search
clf = GridSearchCV(clf, {'anova__percentile': [5, 10, 20]}, cv=cv)
clf.fit(X, y)  # set the best parameters
coef_ = clf.best_estimator_.steps[-1][1].coef_
coef_ = clf.best_estimator_.steps[0][1].inverse_transform(coef_)
coef_selection_ = coef_.reshape(size, size)

###############################################################################
# Inverse the transformation to plot the results on an image
plt.close('all')
plt.figure(figsize=(7.3, 2.7))
plt.subplot(1, 3, 1)
plt.imshow(coef, interpolation="nearest", cmap=plt.cm.RdBu_r)
plt.title("True weights")
plt.subplot(1, 3, 2)
plt.imshow(coef_selection_, interpolation="nearest", cmap=plt.cm.RdBu_r)
plt.title("Feature Selection")
plt.subplot(1, 3, 3)
plt.imshow(coef_agglomeration_, interpolation="nearest", cmap=plt.cm.RdBu_r)
plt.title("Feature Agglomeration")
plt.subplots_adjust(0.04, 0.0, 0.98, 0.94, 0.16, 0.26)
plt.show()

# Attempt to remove the temporary cachedir, but don't worry if it fails
shutil.rmtree(cachedir, ignore_errors=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Make a histogram using a logarithmic scale on X axis

See:

- http://stackoverflow.com/questions/6855710/how-to-have-logarithmic-bins-in-a-python-histogram
"""

import numpy as np
import matplotlib.pyplot as plt

# SETUP #######################################################################

# histtype : [‘bar’ | ‘barstacked’ | ‘step’ | ‘stepfilled’]
HIST_TYPE='bar'
ALPHA=0.5

# MAKE DATA ###################################################################

data = np.random.exponential(size=1000000)
#data = np.abs(np.random.normal(size=1000000) * 10000.)
#data = np.random.chisquare(10, size=1000000)

# INIT FIGURE #################################################################

fig = plt.figure(figsize=(8.0, 6.0))

# AX1 #########################################################################

ax1 = fig.add_subplot(211)

res_tuple = ax1.hist(data,
                     bins=50,
                     histtype=HIST_TYPE,
                     alpha=ALPHA)

ax1.set_title("Normal scale")
ax1.set_xlabel("Value")
ax1.set_ylabel("Count")

# AX2 #########################################################################

ax2 = fig.add_subplot(212)

vmin = np.log10(data.min())
vmax = np.log10(data.max())
bins = np.logspace(vmin, vmax, 50)  # <- make a range from 10**vmin to 10**vmax

print(bins)

res_tuple = ax2.hist(data,
                     bins=bins,
                     histtype=HIST_TYPE,
                     alpha=ALPHA)

ax2.set_xscale("log")               # <- Activate log scale on X axis

ax2.set_title("Log scale")
ax2.set_xlabel("Value")
ax2.set_ylabel("Count")

# SHOW AND SAVE FILE ##########################################################

plt.tight_layout()

plt.savefig("hist_logscale_x.png")
plt.show()


""" This is the Bokeh charts testing interface.

"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 - 2014, Continuum Analytics, Inc. All rights reserved.
#
# Powered by the Bokeh Development Team.
#
# The full license is in the file LICENSE.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from __future__ import absolute_import

from collections import OrderedDict
import unittest

import numpy as np
from numpy.testing import assert_array_equal
import pandas as pd

from bokeh.charts import Line

from bokeh.charts.builder.tests._utils import create_chart

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class TestLine(unittest.TestCase):
    def test_supported_input(self):
        xyvalues = OrderedDict()
        y_python = xyvalues['python'] = [2, 3, 7, 5, 26]
        y_pypy = xyvalues['pypy'] = [12, 33, 47, 15, 126]
        y_jython = xyvalues['jython'] = [22, 43, 10, 25, 26]

        xyvaluesdf = pd.DataFrame(xyvalues)

        for i, _xy in enumerate([xyvalues, xyvaluesdf]):
            hm = create_chart(Line, _xy)
            builder = hm._builders[0]
            self.assertEqual(sorted(builder._groups), sorted(list(xyvalues.keys())))
            assert_array_equal(builder._data['x'], [0, 1, 2, 3, 4])
            assert_array_equal(builder._data['y_python'], y_python)
            assert_array_equal(builder._data['y_pypy'], y_pypy)
            assert_array_equal(builder._data['y_jython'], y_jython)

        lvalues = [[2, 3, 7, 5, 26], [12, 33, 47, 15, 126], [22, 43, 10, 25, 26]]
        for _xy in [lvalues, np.array(lvalues)]:
            hm = create_chart(Line, _xy)
            builder = hm._builders[0]
            self.assertEqual(builder._groups, ['0', '1', '2'])
            assert_array_equal(builder._data['x'], [0, 1, 2, 3, 4])
            assert_array_equal(builder._data['y_0'], y_python)
            assert_array_equal(builder._data['y_1'], y_pypy)
            assert_array_equal(builder._data['y_2'], y_jython)


"""
=======================================================
Comparison of LDA and PCA 2D projection of Iris dataset
=======================================================

The Iris dataset represents 3 kind of Iris flowers (Setosa, Versicolour
and Virginica) with 4 attributes: sepal length, sepal width, petal length
and petal width.

Principal Component Analysis (PCA) applied to this data identifies the
combination of attributes (principal components, or directions in the
feature space) that account for the most variance in the data. Here we
plot the different samples on the 2 first principal components.

Linear Discriminant Analysis (LDA) tries to identify attributes that
account for the most variance *between classes*. In particular,
LDA, in contrast to PCA, is a supervised method, using known class labels.
"""
print(__doc__)

import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.decomposition import PCA
from sklearn.lda import LDA

iris = datasets.load_iris()

X = iris.data
y = iris.target
target_names = iris.target_names

pca = PCA(n_components=2)
X_r = pca.fit(X).transform(X)

lda = LDA(n_components=2)
X_r2 = lda.fit(X, y).transform(X)

# Percentage of variance explained for each components
print('explained variance ratio (first two components): %s'
      % str(pca.explained_variance_ratio_))

plt.figure()
for c, i, target_name in zip("rgb", [0, 1, 2], target_names):
    plt.scatter(X_r[y == i, 0], X_r[y == i, 1], c=c, label=target_name)
plt.legend()
plt.title('PCA of IRIS dataset')

plt.figure()
for c, i, target_name in zip("rgb", [0, 1, 2], target_names):
    plt.scatter(X_r2[y == i, 0], X_r2[y == i, 1], c=c, label=target_name)
plt.legend()
plt.title('LDA of IRIS dataset')

plt.show()


from __future__ import division
import matplotlib.pyplot as plt
import numpy as np
import theano
import theano.tensor as T
import lasagne
from gen_data_029 import gen_data, N_BATCH, LENGTH
theano.config.compute_test_value = 'raise'

# Number of units in the hidden (recurrent) layer
N_HIDDEN = 5
# SGD learning rate
LEARNING_RATE = 1e-1
# Number of iterations to train the net
N_ITERATIONS = 200

# Generate a "validation" sequence whose cost we will periodically compute
X_val, y_val = gen_data()

n_features = X_val.shape[-1]
n_output = y_val.shape[-1]
assert X_val.shape == (N_BATCH, LENGTH, n_features)
assert y_val.shape == (N_BATCH, LENGTH, n_output)

# Construct LSTM RNN: One LSTM layer and one dense output layer
l_in = lasagne.layers.InputLayer(shape=(N_BATCH, LENGTH, n_features))

# setup fwd and bck LSTM layer.
l_fwd = lasagne.layers.LSTMLayer(
    l_in, N_HIDDEN, backwards=False, learn_init=True, peepholes=True)
l_bck = lasagne.layers.LSTMLayer(
    l_in, N_HIDDEN, backwards=True, learn_init=True, peepholes=True)

# concatenate forward and backward LSTM layers
l_fwd_reshape = lasagne.layers.ReshapeLayer(l_fwd, (N_BATCH*LENGTH, N_HIDDEN))
l_bck_reshape = lasagne.layers.ReshapeLayer(l_bck, (N_BATCH*LENGTH, N_HIDDEN))
l_concat = lasagne.layers.ConcatLayer([l_fwd_reshape, l_bck_reshape], axis=1)

l_recurrent_out = lasagne.layers.DenseLayer(
    l_concat, num_units=n_output, nonlinearity=None)
l_out = lasagne.layers.ReshapeLayer(
    l_recurrent_out, (N_BATCH, LENGTH, n_output))

input = T.tensor3('input')
target_output = T.tensor3('target_output')

# add test values
input.tag.test_value = np.random.rand(
    *X_val.shape).astype(theano.config.floatX)
target_output.tag.test_value = np.random.rand(
    *y_val.shape).astype(theano.config.floatX)

# Cost = mean squared error
cost = T.mean((l_out.get_output(input) - target_output)**2)

# Use NAG for training
all_params = lasagne.layers.get_all_params(l_out)
updates = lasagne.updates.nesterov_momentum(cost, all_params, LEARNING_RATE)
# Theano functions for training, getting output, and computing cost
train = theano.function([input, target_output],
                        cost, updates=updates, on_unused_input='warn',
                        allow_input_downcast=True)
y_pred = theano.function(
    [input], l_out.get_output(input), on_unused_input='warn',
    allow_input_downcast=True)

compute_cost = theano.function(
    [input, target_output], cost, on_unused_input='warn',
    allow_input_downcast=True)

# Train the net
def run_training():
    costs = np.zeros(N_ITERATIONS)
    for n in range(N_ITERATIONS):
        X, y = gen_data()

        # you should use your own training data mask instead of mask_val
        costs[n] = train(X, y)
        if not n % 10:
            cost_val = compute_cost(X_val, y_val)
            print "Iteration {} validation cost = {}".format(n, cost_val)

    plt.plot(costs)
    plt.xlabel('Iteration')
    plt.ylabel('Cost')
    plt.show()

def plot_estimates():
    X, y = gen_data()
    y_predictions = y_pred(X)
    ax = plt.gca()
    ax.plot(y_predictions[0,:,0], label='estimate')
    ax.plot(y[0,:,0], label='ground truth')
    # ax.plot(X[0,:,0], label='aggregate')
    ax.legend()
    plt.show()

run_training()
plot_estimates()


#encoding: utf-8

import cPickle as pickle
from classify import load_cls, label_chars
from cv2 import GaussianBlur
from feature_extraction import get_zernike_moments, get_hu_moments, \
    extract_features, normalize_and_extract_features
from functools import partial
import glob
from multiprocessing.pool import Pool
import numpy as np
import os
from sklearn.externals import joblib
from sobel_features import sobel_features
from transitions import transition_features
from fast_utils import fnormalize, ftrim

cls = load_cls('logistic-cls')

# Load testing sets
print 'Loading test data'

tsets = pickle.load(open('datasets/testing/training_sets.pkl', 'rb'))

scaler = joblib.load('zernike_scaler-latest')

print 'importing classifier'

print cls.get_params()

print 'scoring ...'
keys = tsets.keys()
keys.sort()
all_samples = []

## Baseline accuracies for the data in tsets
baseline = [0.608, 0.5785123966942148, 0.4782608695652174, 0.7522123893805309,
            0.6884057971014492, 0.5447154471544715, 0.9752066115702479,
            0.9830508474576272]

def test_accuracy(t, clsf=None):
    '''Get accuracy score for a testset t'''
    if clsf:
        cls = clsf
    else:
        global cls

    y = tsets[t][:,0]
    x = tsets[t][:,1:]

    x3 = []
    for j in x:
        j = ftrim(j.reshape((32,16)).astype(np.uint8))
        x3.append(normalize_and_extract_features(j))

    pred = cls.predict(x3)

    s  = 0
    for i, p in enumerate(pred):
        if float(p) == y[i]:
            s += 1.0
        else:
            pass
            print 'correct', label_chars[y[i]], '||', label_chars[p], t #, max(cls.predict_proba(x3[i])[0])

    score = s / len(y)
    return score

def test_all(clsf=None):
    '''Run accuracy tests for all testsets'''

    print 'starting tests. this will take a moment'

    test_accuracy(keys[0], clsf)

    test_all = partial(test_accuracy, clsf=clsf)
    p = Pool()
    all_samples = p.map(test_all, keys)

    for t, s in zip(keys, all_samples):
        print t, s
    return np.mean(all_samples)

if __name__ == '__main__':
    print test_all()


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Andrés Aguirre Dorelo
#
# MINA/INCO/UDELAR
#
# module for finding the steps in the tutors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import os
import glob
import ntpath

from parser.BvhImport import BvhImport
import matplotlib.pyplot as plt
from configuration.LoadSystemConfiguration import LoadSystemConfiguration
import numpy as np
from scipy.signal import argrelextrema
from collections import Counter

sysConf = LoadSystemConfiguration()
BVHDir = os.getcwd() + sysConf.getDirectory("CMU mocap Files")

Y_THREADHOLD = 11 #TODO calculate this as the average of the steps_highs
X_THREADHOLD = 36

def firstMax(values1, values2):
    res=0
    for i in range(len(values1)-2):
        if values1[i] < values1[i+1] and values1[i+1] > values1[i+2]: #i+1 is a local maximun
            if (values1[i] - values2[i]) > THREADHOLD:
                res=i+1
        elif values1[i] < values1[i+1] < values1[i+2]: #i is a local maximun
            if (values1[i] - values2[i]) > THREADHOLD:
                res=i
    return res

def find_nearest(a, a0):
    "Element in nd array `a` closest to the scalar value `a0`"
    idx = np.abs(a - a0).argmin()
    return a.flat[idx]

for filename in glob.glob(os.path.join(BVHDir, '*.bvh')):
    print "transforming: " + filename + " ..."
    parser = BvhImport(filename)
    x_,y_,z_ = parser.getNodePositionsFromName("lFoot")
    y1 = []
    y2 = []
    x1 = []
    x2 = []

    for key, value in y_.iteritems():
        y1.append(value)
        x1.append(key)

    x_,y_,z_ = parser.getNodePositionsFromName("rFoot")
    for key, value in y_.iteritems():
        y2.append(value)
        x2.append(key)

    maxLfootIndexes = [x for x in argrelextrema(np.array(y1), np.greater)[0]]
    maxRfootIndexes = [x for x in argrelextrema(np.array(y2), np.greater)[0]]

    stepsLfootIndexes = []
    for i in range(len(maxLfootIndexes)):
        index = maxLfootIndexes[i]
        if y1[index] - y2[index] > Y_THREADHOLD: #one foot is up and the other is in the floor
                if len(stepsLfootIndexes)>0:
                    if abs(index - find_nearest(np.array(stepsLfootIndexes), index) > X_THREADHOLD): #avoid max near an existing point
                        stepsLfootIndexes.append(index)
                        print "appeend L"
                    else:
                        if y1[find_nearest(np.array(stepsLfootIndexes), index)] < y1[index]:  #check if the exiting near max is a local maximun
                            print "remove L", find_nearest(np.array(stepsLfootIndexes), index), "from: ", stepsLfootIndexes
                            stepsLfootIndexes.remove(find_nearest(np.array(stepsLfootIndexes), index))
                            print "remove L"
                            stepsLfootIndexes.append(index)
                            print "appeend L"
                else:
                    stepsLfootIndexes.append(index)
                    print "appeend L"

    stepsRfootIndexes = []
    for i in range(len(maxRfootIndexes)):
        index = maxRfootIndexes[i]
        if y2[index] - y1[index] > Y_THREADHOLD: #one foot is up and the other is in the floor
                if len(stepsRfootIndexes)>0:
                    if abs(index - find_nearest(np.array(stepsRfootIndexes),index) > X_THREADHOLD): #avoid max near an existing point
                        stepsRfootIndexes.append(index)
                        print "appeend R"
                    else:
                        if y2[find_nearest(np.array(stepsRfootIndexes), index)] < y2[index]: #check if the exiting near max is a local maximun
                            print "remove R", find_nearest(np.array(stepsRfootIndexes), index), "from: ", stepsRfootIndexes, "index: ", index
                            stepsRfootIndexes.remove(find_nearest(np.array(stepsRfootIndexes), index))
                            print "remove R"
                            stepsRfootIndexes.append(index)
                            print "appeend R"

                else:
                    stepsRfootIndexes.append(index)
                    print "appeend R"

    if stepsLfootIndexes[0] < stepsRfootIndexes[0]:
        if len(stepsLfootIndexes) > 2:
            testPoint = stepsLfootIndexes[1]
            while(y1[testPoint]>y2[testPoint]):
                testPoint = testPoint + 1

            end = testPoint + 5
            print "red over green| ", "red: ", stepsLfootIndexes[0], "green: ", stepsRfootIndexes[0], "second red: ", stepsLfootIndexes[1], "end: ", end
        else:
            end = len(y1)
            print "red over green| ", "red: ", stepsLfootIndexes[0], "green: ", stepsRfootIndexes[0], "second red: -----", "end: ", end

    else:
        if len(stepsRfootIndexes) > 2:
            testPoint = stepsRfootIndexes[1]
            while(y2[testPoint]>y1[testPoint]):
                testPoint = testPoint + 1
            end = testPoint + 5
            print "green over red| ", "green: ", stepsRfootIndexes[0], "red: ", stepsLfootIndexes[0], "second green: ", stepsRfootIndexes[1], "end: ", end
        else:
            end = len(y2)
            print "green over red| ", "green: ", stepsRfootIndexes[0], "red: ", stepsLfootIndexes[0], "second green: -----", "end: ", end

    plt.plot(x1, y1,'ro')
    plt.plot(x1, y2,'g')
    plt.show()


#
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pandas as pd
import pytz

from datetime import datetime
from dateutil import rrule
from zipline.utils.tradingcalendar import end, canonicalize_datetime, \
    get_open_and_closes

start = pd.Timestamp('1994-01-01', tz='UTC')

def get_non_trading_days(start, end):
    non_trading_rules = []

    start = canonicalize_datetime(start)
    end = canonicalize_datetime(end)

    weekends = rrule.rrule(
        rrule.YEARLY,
        byweekday=(rrule.SA, rrule.SU),
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(weekends)

    # Universal confraternization
    conf_universal = rrule.rrule(
        rrule.MONTHLY,
        byyearday=1,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(conf_universal)

    # Sao Paulo city birthday
    aniversario_sao_paulo = rrule.rrule(
        rrule.MONTHLY,
        bymonth=1,
        bymonthday=25,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(aniversario_sao_paulo)

    # Carnival Monday
    carnaval_segunda = rrule.rrule(
        rrule.MONTHLY,
        byeaster=-48,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(carnaval_segunda)

    # Carnival Tuesday
    carnaval_terca = rrule.rrule(
        rrule.MONTHLY,
        byeaster=-47,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(carnaval_terca)

    # Passion of the Christ
    sexta_paixao = rrule.rrule(
        rrule.MONTHLY,
        byeaster=-2,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(sexta_paixao)

    # Corpus Christi
    corpus_christi = rrule.rrule(
        rrule.MONTHLY,
        byeaster=60,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(corpus_christi)

    tiradentes = rrule.rrule(
        rrule.MONTHLY,
        bymonth=4,
        bymonthday=21,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(tiradentes)

    # Labor day
    dia_trabalho = rrule.rrule(
        rrule.MONTHLY,
        bymonth=5,
        bymonthday=1,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(dia_trabalho)

    # Constitutionalist Revolution
    constitucionalista = rrule.rrule(
        rrule.MONTHLY,
        bymonth=7,
        bymonthday=9,
        cache=True,
        dtstart=datetime(1997, 1, 1, tzinfo=pytz.utc),
        until=end
    )
    non_trading_rules.append(constitucionalista)

    # Independency day
    independencia = rrule.rrule(
        rrule.MONTHLY,
        bymonth=9,
        bymonthday=7,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(independencia)

    # Our Lady of Aparecida
    aparecida = rrule.rrule(
        rrule.MONTHLY,
        bymonth=10,
        bymonthday=12,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(aparecida)

    # All Souls' day
    finados = rrule.rrule(
        rrule.MONTHLY,
        bymonth=11,
        bymonthday=2,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(finados)

    # Proclamation of the Republic
    proclamacao_republica = rrule.rrule(
        rrule.MONTHLY,
        bymonth=11,
        bymonthday=15,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(proclamacao_republica)

    # Day of Black Awareness
    consciencia_negra = rrule.rrule(
        rrule.MONTHLY,
        bymonth=11,
        bymonthday=20,
        cache=True,
        dtstart=datetime(2004, 1, 1, tzinfo=pytz.utc),
        until=end
    )
    non_trading_rules.append(consciencia_negra)

    # Christmas Eve
    vespera_natal = rrule.rrule(
        rrule.MONTHLY,
        bymonth=12,
        bymonthday=24,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(vespera_natal)

    # Christmas
    natal = rrule.rrule(
        rrule.MONTHLY,
        bymonth=12,
        bymonthday=25,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(natal)

    # New Year Eve
    ano_novo = rrule.rrule(
        rrule.MONTHLY,
        bymonth=12,
        bymonthday=31,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(ano_novo)

    # New Year Eve on saturday
    ano_novo_sab = rrule.rrule(
        rrule.MONTHLY,
        bymonth=12,
        bymonthday=30,
        byweekday=rrule.FR,
        cache=True,
        dtstart=start,
        until=end
    )
    non_trading_rules.append(ano_novo_sab)

    non_trading_ruleset = rrule.rruleset()

    for rule in non_trading_rules:
        non_trading_ruleset.rrule(rule)

    non_trading_days = non_trading_ruleset.between(start, end, inc=True)

    # World Cup 2014 Opening
    non_trading_days.append(datetime(2014, 6, 12, tzinfo=pytz.utc))

    non_trading_days.sort()
    return pd.DatetimeIndex(non_trading_days)

non_trading_days = get_non_trading_days(start, end)
trading_day = pd.tseries.offsets.CDay(holidays=non_trading_days)

def get_trading_days(start, end, trading_day=trading_day):
    return pd.date_range(start=start.date(),
                         end=end.date(),
                         freq=trading_day).tz_localize('UTC')

trading_days = get_trading_days(start, end)

# Ash Wednesday
quarta_cinzas = rrule.rrule(
    rrule.MONTHLY,
    byeaster=-46,
    cache=True,
    dtstart=start,
    until=end
)

def get_early_closes(start, end):
    # TSX closed at 1:00 PM on december 24th.

    start = canonicalize_datetime(start)
    end = canonicalize_datetime(end)

    early_close_rules = []

    early_close_rules.append(quarta_cinzas)

    early_close_ruleset = rrule.rruleset()

    for rule in early_close_rules:
        early_close_ruleset.rrule(rule)
    early_closes = early_close_ruleset.between(start, end, inc=True)

    early_closes.sort()
    return pd.DatetimeIndex(early_closes)

early_closes = get_early_closes(start, end)

def get_open_and_close(day, early_closes):
    # only "early close" event in Bovespa actually is a late start
    # as the market only opens at 1pm
    open_hour = 13 if day in quarta_cinzas else 10
    market_open = pd.Timestamp(
        datetime(
            year=day.year,
            month=day.month,
            day=day.day,
            hour=open_hour,
            minute=00),
        tz='America/Sao_Paulo').tz_convert('UTC')
    market_close = pd.Timestamp(
        datetime(
            year=day.year,
            month=day.month,
            day=day.day,
            hour=16),
        tz='America/Sao_Paulo').tz_convert('UTC')

    return market_open, market_close

open_and_closes = get_open_and_closes(trading_days, early_closes,
                                      get_open_and_close)


"""
===================================================
Recursive feature elimination with cross-validation
===================================================

A recursive feature elimination example with automatic tuning of the
number of features selected with cross-validation.
"""
print(__doc__)

import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.cross_validation import StratifiedKFold
from sklearn.feature_selection import RFECV
from sklearn.datasets import make_classification

# Build a classification task using 3 informative features
X, y = make_classification(n_samples=1000, n_features=25, n_informative=3,
                           n_redundant=2, n_repeated=0, n_classes=8,
                           n_clusters_per_class=1, random_state=0)

# Create the RFE object and compute a cross-validated score.
svc = SVC(kernel="linear")
# The "accuracy" scoring is proportional to the number of correct
# classifications
rfecv = RFECV(estimator=svc, step=1, cv=StratifiedKFold(y, 2),
              scoring='accuracy')
rfecv.fit(X, y)

print("Optimal number of features : %d" % rfecv.n_features_)

# Plot number of features VS. cross-validation scores
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score (nb of correct classifications)")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()


import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits.mplot3d.art3d import Line3D
from matplotlib.lines import Line2D
import numpy as np

def window(name):
    return plt.figure(name)

def show():
    plt.show()
    return None

def undeformed(model):
    """Plot the undeformed structure according to the dimension

    """
    if model.ndm == 2:
        undeformed = window('Undeformed')
        axes = undeformed.add_subplot(111, aspect='equal')
        geo2d(model.XYZ, model.CON, axes, color='black')
        label2d(model.XYZ, model.CON, axes)
        undeformed.tight_layout()

    if model.ndm == 3:
        undeformed = window('Undeformed')
        axes = undeformed.add_subplot(111, projection='3d', aspect='equal')
        geo3d(model.XYZ, model.CON, axes, 'black')
        label3d(model.XYZ, model.CON, axes)
        undeformed.tight_layout()

def deformed(model, U):
    """Plot the deformed structure according to the dimension

    """
    CON = model.CON

    XYZ = np.copy(model.XYZ)
    for n in range(model.nn):
        for d in range(model.ndf[n]):
            dof = model.DOF[n, d]
            XYZ[n, d] += U[dof]

    if model.ndm == 2:
        deformed = window('Deformed')
        axes = deformed.add_subplot(111, aspect='equal')
        geo2d(XYZ, CON, axes, 'tomato')
        geo2d(model.XYZ, model.CON, axes, 'black')
        label2d(XYZ, CON, axes)
        deformed.tight_layout()

    if model.ndm == 3:
        deformed = window('Deformed')
        axes = deformed.add_subplot(111, projection='3d', aspect='equal')
        geo3d(model.XYZ, model.CON, axes, 'black')
        geo3d(XYZ, CON, axes, 'tomato')
        label3d(XYZ, CON, axes)
        deformed.tight_layout()

def geo3d(XYZ, CON, axes, color):
    """Plot the 3d model

    """
    axes.set_xlabel('x')
    axes.set_ylabel('y')
    axes.set_zlabel('z')

    # draw nodes
    for node, xyz in enumerate(XYZ):
        axes.scatter(xyz[0], xyz[1], xyz[2], c='k', alpha=1, marker='s')

    # draw edges
    for ele, con in enumerate(CON):
        xs = [XYZ[con[0]][0], XYZ[con[1]][0]]
        ys = [XYZ[con[0]][1], XYZ[con[1]][1]]
        zs = [XYZ[con[0]][2], XYZ[con[1]][2]]
        line = Line3D(xs, ys, zs, linewidth=1.0, color=color)
        axes.add_line(line)

def label3d(XYZ, CON, axes):
    """Plot the nodes and element label

    """
    for node, xyz in enumerate(XYZ):
        axes.text(xyz[0], xyz[1], xyz[2], str(node), color='b', size=10)

    for ele, con in enumerate(CON):
        xm = (XYZ[con[0]][0] + XYZ[con[1]][0])/2
        ym = (XYZ[con[0]][1] + XYZ[con[1]][1])/2
        zm = (XYZ[con[0]][2] + XYZ[con[1]][2])/2
        axes.text(xm, ym, zm, str(ele), color='g', size=10)

def geo2d(XYZ, CON, axes, color):
    """Plot the 2d model

    """
    axes.set_xlabel('x')
    axes.set_ylabel('y')

    # draw nodes
    for xyz in XYZ:
        axes.scatter(xyz[0], xyz[1], c='k', alpha=1, marker='s')

    # draw edges
    for con in CON:
        xs = [XYZ[con[0]][0], XYZ[con[1]][0]]
        ys = [XYZ[con[0]][1], XYZ[con[1]][1]]
        line = Line2D(xs, ys, linewidth=1.0, color=color)
        axes.add_line(line)

def label2d(XYZ, CON, axes):
    """Plot the nodes and element label

    """
    for node, xyz in enumerate(XYZ):
        axes.text(xyz[0], xyz[1], str(node), color='b', size=10)

    for ele, con in enumerate(CON):
        xm = (XYZ[con[0]][0] + XYZ[con[1]][0])/2
        ym = (XYZ[con[0]][1] + XYZ[con[1]][1])/2
        axes.text(xm, ym, str(ele), color='g', size=10)

def axialforce(model, Q):
    """Plot axial force

    """
    if model.ndm == 2:
        axial = window('Axial')
        axes = axial.add_subplot(111, aspect='equal')
        geo2d(model.XYZ, model.CON, axes, color='black')
        axial2d(model.XYZ, model.CON, Q, axes)
        axial.tight_layout()

    if model.ndm == 3:
        axial = window('Axial')
        axes = axial.add_subplot(111, projection='3d', aspect='equal')
        geo3d(model.XYZ, model.CON, axes, 'black')
        axial3d(model.XYZ, model.CON, Q, axes)
        axial.tight_layout()

def axial2d(XYZ, CON, Q, axes):
    """Plot text with axial force value

    """
    for ele, con in enumerate(CON):
        xm = (XYZ[con[0]][0] + XYZ[con[1]][0])/2
        ym = (XYZ[con[0]][1] + XYZ[con[1]][1])/2
        axes.text(xm, ym, str(np.round_(Q[ele], 1)), color='g', size=10)

def axial3d(XYZ, CON, Q, axes):
    """Plot text with axial force value for 3d plot

    """
    for ele, con in enumerate(CON):
        xm = (XYZ[con[0]][0] + XYZ[con[1]][0])/2
        ym = (XYZ[con[0]][1] + XYZ[con[1]][1])/2
        zm = (XYZ[con[0]][2] + XYZ[con[1]][2])/2
        axes.text(xm, ym, zm, str(np.round_(Q[ele], 1)), color='g', size=10)


from mpl_toolkits.basemap import Basemap
import numpy as np
import matplotlib.pyplot as plt
# setup oblique mercator basemap.
# width is width of map projection region in km (xmax-xmin_
# height is height of map projection region in km (ymax-ymin)
# lon_0, lat_0 are the central longitude and latitude of the projection.
# lat_1,lon_1 and lat_2,lon_2  are two pairs of points that define
# the projection centerline.
# Map projection coordinates are automatically rotated to true north.
# To avoid this, set no_rot=True.
# area_thresh=1000 means don't plot coastline features less
# than 1000 km^2 in area.
m = Basemap(height=16700000,width=12000000,
            resolution='l',area_thresh=1000.,projection='omerc',\
            lon_0=-100,lat_0=15,lon_2=-120,lat_2=65,lon_1=-50,lat_1=-55)
m.drawcoastlines()
m.fillcontinents(color='coral',lake_color='aqua')
# draw parallels and meridians.
m.drawparallels(np.arange(-80.,81.,20.))
m.drawmeridians(np.arange(-180.,181.,20.))
m.drawmapboundary(fill_color='aqua')
plt.title("Oblique Mercator Projection")
plt.show()