import cv2
import sys
import pickle
import numpy as np
import matplotlib.pyplot as plt

from AffineInvariantFeatures import AffineInvariant
from TemplateMatcher import TemplateMatch, Templates

from PIL import Image
from itertools import izip_longest

from sklearn.cross_validation import train_test_split
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import classification_report, roc_curve, auc, confusion_matrix, accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.tree import export_graphviz, DecisionTreeClassifier

def line_count(filename):
    with open(filename) as data:
        return sum(1 for line in data)

def read_image(filename):
    return np.array(Image.open(filename.strip('\n')).convert('L'), np.uint8)

def read_file(filename, limit=0):
    n = 0
    lines = line_count(filename)

    with open(filename) as data:
        while True:
            line = next(data, None)
            if not line or (limit and n >= limit):
                break

            n += 1
            print '\r%s %d/%d' % (filename, n, limit or lines),
            try:
                yield read_image(line)
            except:
                continue

def get_templates():
    return np.array(list(read_file('templates.txt')))

def get_images(limit=0):
    positive = read_file('positive.txt', limit / 2 if limit else 0)
    negative = read_file('negative.txt', limit / 2 if limit else 0)

    for p, n in izip_longest(positive, negative):
        if p is not None:
            yield (1, p)
        if n is not None:
            yield (0, n)

def get_dataset(limit):
    return map(np.asarray, zip(*get_images(limit)))

def plot_roc(fpr, tpr, roc_auc):
    # Plot all ROC curves
    plt.figure()

    plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)

    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Affine Invariant SURF + Decision Tree Classifier')
    plt.legend(loc='lower right')
    plt.show()

def plot_importance(feature_count, importances, indices):
    plt.figure()
    plt.title('Feature importances')
    plt.bar(range(feature_count), importances[indices], color='r', align='center')
    plt.xticks(range(feature_count), indices)
    plt.xlim([-1, feature_count])
    plt.show()

def main(name, dataset_size):
    templates = get_templates()

    print 'templates: %d' % len(templates)

    labels, samples = get_dataset(dataset_size)

    print 'samples: %d' % len(samples)

    extractor = cv2.FeatureDetector_create('SURF')
    detector = cv2.DescriptorExtractor_create('SURF')

    print 'applying affine invariant transform'

    affine = AffineInvariant(extractor, detector)
    templates = affine.transform(templates)
    samples = affine.transform(samples)

    model = Pipeline([
        ('match', TemplateMatch(Templates(templates))),  # XXX: hack to bypass cloning error
        # ('reduce_dim', PCA(n_components = 12 * 6))
    ])

    samples = model.fit_transform(samples)

    rng = np.random.RandomState()

    X_train, X_test, y_train, y_test = train_test_split(samples, labels, test_size=0.5, random_state=rng)
    print 'train: %d, test: %d' % (len(X_train), len(X_test))

    params = dict(
        min_samples_split = [5, 6, 7, 8, 9, 10],
        min_samples_leaf = [3, 4, 5, 6, 7],
        max_leaf_nodes = [10, 9, 8, 7, 6],
        class_weight = [{1: w} for w in [10, 8, 4, 2, 1]]
    )

    tree = DecisionTreeClassifier(max_depth=4, random_state=rng)

    cvmodel = GridSearchCV(tree, params, cv=10, n_jobs=cv2.getNumberOfCPUs())
    cvmodel.fit(X_train, y_train)

    print 'grid scores'
    for params, mean_score, scores in cvmodel.grid_scores_:
        print '%0.3f (+/-%0.03f) for %r' % (mean_score, scores.std() * 2, params)
    print 'best parameters'
    print cvmodel.best_params_

    importances = cvmodel.best_estimator_.feature_importances_
    indices = np.argsort(importances)[::-1]

    plot_importance(6, importances, indices)

    y_pred = cvmodel.predict(X_test)
    accuracy = accuracy_score(y_test, y_pred)

    print 'accuracy: %f' % accuracy
    print classification_report(y_test, y_pred)
    print confusion_matrix(y_test, y_pred)

    y_score = cvmodel.predict_proba(X_test)[:, 1]
    fpr, tpr, _ = roc_curve(y_test, y_score)
    roc_auc = auc(fpr, tpr)

    plot_roc(fpr, tpr, roc_auc)

    export_graphviz(cvmodel.best_estimator_, out_file=name + '.dot', class_names=['background', 'badge'], filled=True, rounded=True, special_characters=True)
    pickle.dump(dict(params=params, pipe=model, model=cvmodel.best_estimator_), open(name + '.pkl', 'wb'))

if __name__ == '__main__':
    name = sys.argv[1] if len(sys.argv) >= 2 else 'classifier'
    dataset_size = int(sys.argv[2]) if len(sys.argv) >= 3 else 0

    main(name, dataset_size)


'''function [out] = imrotateFill(inp, angle)
% function  [out] = imrotateFill(inp)
% Rotates an 2D image couterclockwise by angle in degrees
% Output image has the same dimension as input.
% Undefined regions are filled in by repeating the original image
% Note: input images must be square
%
% Copyright (c) UWM, Peter Schwander Mar. 20, 2014
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
version = 'imrotateFill, V0.9';

Ported to python. Hstau Liao Oct. 2016
'''

import numpy as np
import logging,sys
import math
from scipy.ndimage.interpolation import rotate
import matplotlib.pyplot as plt

def op(input, angle, visual=False):
    nPix = input.shape[0]
    inpRep = np.tile(input, (3, 3))
    outRep = rotate(inpRep, angle, reshape=False)
    out = outRep[nPix:2 * nPix, nPix:2 * nPix]

    if visual:
        plt.subplot(2, 2, 1)
        plt.imshow(input,cmap = plt.get_cmap('gray'))
        plt.title('Input')
        plt.subplot(2, 2, 2)
        plt.imshow(out, cmap=plt.get_cmap('gray'))
        plt.title('Output')
        plt.subplot(2, 2, 3)
        plt.imshow(inpRep, cmap=plt.get_cmap('gray'))
        plt.title('Input 3x3')
        plt.subplot(2, 2, 4)
        plt.imshow(outRep, cmap=plt.get_cmap('gray'))
        plt.title('Output 3x3')
        plt.show()
    return out

if __name__ == '__main__':

    # tested using a 6x6 image
    img = np.loadtxt(sys.argv[1])
    ang = float(sys.argv[2]) # in degrees
    visual = bool(sys.argv[3])
    result = op(img,ang,visual)


"""
==================================================
Compute the power spectral density of raw data
==================================================

This script shows how to compute the power spectral density (PSD)
of measurements on a raw dataset. It also show the effect of applying SSP
to the data to reduce ECG and EOG artifacts.
"""
# Authors: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#          Martin Luessi <mluessi@nmr.mgh.harvard.edu>
#          Eric Larson <larson.eric.d@gmail.com>
# License: BSD (3-clause)

import numpy as np
import matplotlib.pyplot as plt

import mne
from mne import io, read_proj, read_selection
from mne.datasets import sample
from mne.time_frequency import psd_multitaper

print(__doc__)

###############################################################################
# Set parameters
data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_raw.fif'
proj_fname = data_path + '/MEG/sample/sample_audvis_eog-proj.fif'

tmin, tmax = 0, 60  # use the first 60s of data

# Setup for reading the raw data (to save memory, crop before loading)
raw = io.read_raw_fif(raw_fname).crop(tmin, tmax).load_data()
raw.info['bads'] += ['MEG 2443', 'EEG 053']  # bads + 2 more

# Add SSP projection vectors to reduce EOG and ECG artifacts
projs = read_proj(proj_fname)
raw.add_proj(projs, remove_existing=True)

fmin, fmax = 2, 300  # look at frequencies between 2 and 300Hz
n_fft = 2048  # the FFT size (n_fft). Ideally a power of 2

# Let's first check out all channel types
raw.plot_psd(area_mode='range', tmax=10.0, show=False)

# Now let's focus on a smaller subset:
# Pick MEG magnetometers in the Left-temporal region
selection = read_selection('Left-temporal')
picks = mne.pick_types(raw.info, meg='mag', eeg=False, eog=False,
                       stim=False, exclude='bads', selection=selection)

# Let's just look at the first few channels for demonstration purposes
picks = picks[:4]

plt.figure()
ax = plt.axes()
raw.plot_psd(tmin=tmin, tmax=tmax, fmin=fmin, fmax=fmax, n_fft=n_fft,
             n_jobs=1, proj=False, ax=ax, color=(0, 0, 1),  picks=picks,
             show=False)

# And now do the same with SSP applied
raw.plot_psd(tmin=tmin, tmax=tmax, fmin=fmin, fmax=fmax, n_fft=n_fft,
             n_jobs=1, proj=True, ax=ax, color=(0, 1, 0), picks=picks,
             show=False)

# And now do the same with SSP + notch filtering
# Pick all channels for notch since the SSP projection mixes channels together
raw.notch_filter(np.arange(60, 241, 60), n_jobs=1)
raw.plot_psd(tmin=tmin, tmax=tmax, fmin=fmin, fmax=fmax, n_fft=n_fft,
             n_jobs=1, proj=True, ax=ax, color=(1, 0, 0), picks=picks,
             show=False)

ax.set_title('Four left-temporal magnetometers')
plt.legend(['Without SSP', 'With SSP', 'SSP + Notch'])

# Alternatively, you may also create PSDs from Raw objects with ``psd_*``
f, ax = plt.subplots()
psds, freqs = psd_multitaper(raw, low_bias=True, tmin=tmin, tmax=tmax,
                             fmin=fmin, fmax=fmax, proj=True, picks=picks,
                             n_jobs=1)
psds = 10 * np.log10(psds)
psds_mean = psds.mean(0)
psds_std = psds.std(0)

ax.plot(freqs, psds_mean, color='k')
ax.fill_between(freqs, psds_mean - psds_std, psds_mean + psds_std,
                color='k', alpha=.5)
ax.set(title='Multitaper PSD', xlabel='Frequency',
       ylabel='Power Spectral Density (dB)')
plt.show()


"""Tests for plotting utilities."""
import warnings
import tempfile
import shutil

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import nose
import nose.tools as nt
from nose.tools import assert_equal, raises
import numpy.testing as npt
import pandas.util.testing as pdt

from distutils.version import LooseVersion
pandas_has_categoricals = LooseVersion(pd.__version__) >= "0.15"

from pandas.util.testing import network

try:
    from bs4 import BeautifulSoup
except ImportError:
    BeautifulSoup = None

from . import PlotTestCase
from .. import utils, rcmod
from ..utils import get_dataset_names, load_dataset

a_norm = np.random.randn(100)

def test_pmf_hist_basics():
    """Test the function to return barplot args for pmf hist."""
    out = utils.pmf_hist(a_norm)
    assert_equal(len(out), 3)
    x, h, w = out
    assert_equal(len(x), len(h))

    # Test simple case
    a = np.arange(10)
    x, h, w = utils.pmf_hist(a, 10)
    nose.tools.assert_true(np.all(h == h[0]))

def test_pmf_hist_widths():
    """Test histogram width is correct."""
    x, h, w = utils.pmf_hist(a_norm)
    assert_equal(x[1] - x[0], w)

def test_pmf_hist_normalization():
    """Test that output data behaves like a PMF."""
    x, h, w = utils.pmf_hist(a_norm)
    nose.tools.assert_almost_equal(sum(h), 1)
    nose.tools.assert_less_equal(h.max(), 1)

def test_pmf_hist_bins():
    """Test bin specification."""
    x, h, w = utils.pmf_hist(a_norm, 20)
    assert_equal(len(x), 20)

def test_ci_to_errsize():
    """Test behavior of ci_to_errsize."""
    cis = [[.5, .5],
           [1.25, 1.5]]

    heights = [1, 1.5]

    actual_errsize = np.array([[.5, 1],
                               [.25, 0]])

    test_errsize = utils.ci_to_errsize(cis, heights)
    npt.assert_array_equal(actual_errsize, test_errsize)

def test_desaturate():
    """Test color desaturation."""
    out1 = utils.desaturate("red", .5)
    assert_equal(out1, (.75, .25, .25))

    out2 = utils.desaturate("#00FF00", .5)
    assert_equal(out2, (.25, .75, .25))

    out3 = utils.desaturate((0, 0, 1), .5)
    assert_equal(out3, (.25, .25, .75))

    out4 = utils.desaturate("red", .5)
    assert_equal(out4, (.75, .25, .25))

@raises(ValueError)
def test_desaturation_prop():
    """Test that pct outside of [0, 1] raises exception."""
    utils.desaturate("blue", 50)

def test_saturate():
    """Test performance of saturation function."""
    out = utils.saturate((.75, .25, .25))
    assert_equal(out, (1, 0, 0))

def test_iqr():
    """Test the IQR function."""
    a = np.arange(5)
    iqr = utils.iqr(a)
    assert_equal(iqr, 2)

class TestSpineUtils(PlotTestCase):

    sides = ["left", "right", "bottom", "top"]
    outer_sides = ["top", "right"]
    inner_sides = ["left", "bottom"]

    offset = 10
    original_position = ("outward", 0)
    offset_position = ("outward", offset)

    def test_despine(self):
        f, ax = plt.subplots()
        for side in self.sides:
            nt.assert_true(ax.spines[side].get_visible())

        utils.despine()
        for side in self.outer_sides:
            nt.assert_true(~ax.spines[side].get_visible())
        for side in self.inner_sides:
            nt.assert_true(ax.spines[side].get_visible())

        utils.despine(**dict(zip(self.sides, [True] * 4)))
        for side in self.sides:
            nt.assert_true(~ax.spines[side].get_visible())

    def test_despine_specific_axes(self):
        f, (ax1, ax2) = plt.subplots(2, 1)

        utils.despine(ax=ax2)

        for side in self.sides:
            nt.assert_true(ax1.spines[side].get_visible())

        for side in self.outer_sides:
            nt.assert_true(~ax2.spines[side].get_visible())
        for side in self.inner_sides:
            nt.assert_true(ax2.spines[side].get_visible())

    def test_despine_with_offset(self):
        f, ax = plt.subplots()

        for side in self.sides:
            nt.assert_equal(ax.spines[side].get_position(),
                            self.original_position)

        utils.despine(ax=ax, offset=self.offset)

        for side in self.sides:
            is_visible = ax.spines[side].get_visible()
            new_position = ax.spines[side].get_position()
            if is_visible:
                nt.assert_equal(new_position, self.offset_position)
            else:
                nt.assert_equal(new_position, self.original_position)

    def test_despine_with_offset_specific_axes(self):
        f, (ax1, ax2) = plt.subplots(2, 1)

        utils.despine(offset=self.offset, ax=ax2)

        for side in self.sides:
            nt.assert_equal(ax1.spines[side].get_position(),
                            self.original_position)
            if ax2.spines[side].get_visible():
                nt.assert_equal(ax2.spines[side].get_position(),
                                self.offset_position)
            else:
                nt.assert_equal(ax2.spines[side].get_position(),
                                self.original_position)

    def test_despine_trim_spines(self):
        f, ax = plt.subplots()
        ax.plot([1, 2, 3], [1, 2, 3])
        ax.set_xlim(.75, 3.25)

        utils.despine(trim=True)
        for side in self.inner_sides:
            bounds = ax.spines[side].get_bounds()
            nt.assert_equal(bounds, (1, 3))

    def test_despine_trim_inverted(self):

        f, ax = plt.subplots()
        ax.plot([1, 2, 3], [1, 2, 3])
        ax.set_ylim(.85, 3.15)
        ax.invert_yaxis()

        utils.despine(trim=True)
        for side in self.inner_sides:
            bounds = ax.spines[side].get_bounds()
            nt.assert_equal(bounds, (1, 3))

    def test_despine_trim_noticks(self):

        f, ax = plt.subplots()
        ax.plot([1, 2, 3], [1, 2, 3])
        ax.set_yticks([])
        utils.despine(trim=True)
        nt.assert_equal(ax.get_yticks().size, 0)

    def test_offset_spines_warns(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always", category=UserWarning)

            f, ax = plt.subplots()
            utils.offset_spines(offset=self.offset)
            nt.assert_true('deprecated' in str(w[0].message))
            nt.assert_true(issubclass(w[0].category, UserWarning))

    def test_offset_spines(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always", category=UserWarning)
            f, ax = plt.subplots()

            for side in self.sides:
                nt.assert_equal(ax.spines[side].get_position(),
                                self.original_position)

            utils.offset_spines(offset=self.offset)

            for side in self.sides:
                nt.assert_equal(ax.spines[side].get_position(),
                                self.offset_position)

    def test_offset_spines_specific_axes(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always", category=UserWarning)
            f, (ax1, ax2) = plt.subplots(2, 1)

            utils.offset_spines(offset=self.offset, ax=ax2)

            for side in self.sides:
                nt.assert_equal(ax1.spines[side].get_position(),
                                self.original_position)
                nt.assert_equal(ax2.spines[side].get_position(),
                                self.offset_position)

def test_ticklabels_overlap():

    rcmod.set()
    f, ax = plt.subplots(figsize=(2, 2))
    f.tight_layout()  # This gets the Agg renderer working

    assert not utils.axis_ticklabels_overlap(ax.get_xticklabels())

    big_strings = "abcdefgh", "ijklmnop"
    ax.set_xlim(-.5, 1.5)
    ax.set_xticks([0, 1])
    ax.set_xticklabels(big_strings)

    assert utils.axis_ticklabels_overlap(ax.get_xticklabels())

    x, y = utils.axes_ticklabels_overlap(ax)
    assert x
    assert not y

def test_categorical_order():

    x = ["a", "c", "c", "b", "a", "d"]
    y = [3, 2, 5, 1, 4]
    order = ["a", "b", "c", "d"]

    out = utils.categorical_order(x)
    nt.assert_equal(out, ["a", "c", "b", "d"])

    out = utils.categorical_order(x, order)
    nt.assert_equal(out, order)

    out = utils.categorical_order(x, ["b", "a"])
    nt.assert_equal(out, ["b", "a"])

    out = utils.categorical_order(np.array(x))
    nt.assert_equal(out, ["a", "c", "b", "d"])

    out = utils.categorical_order(pd.Series(x))
    nt.assert_equal(out, ["a", "c", "b", "d"])

    out = utils.categorical_order(y)
    nt.assert_equal(out, [1, 2, 3, 4, 5])

    out = utils.categorical_order(np.array(y))
    nt.assert_equal(out, [1, 2, 3, 4, 5])

    out = utils.categorical_order(pd.Series(y))
    nt.assert_equal(out, [1, 2, 3, 4, 5])

    if pandas_has_categoricals:
        x = pd.Categorical(x, order)
        out = utils.categorical_order(x)
        nt.assert_equal(out, list(x.categories))

        x = pd.Series(x)
        out = utils.categorical_order(x)
        nt.assert_equal(out, list(x.cat.categories))

        out = utils.categorical_order(x, ["b", "a"])
        nt.assert_equal(out, ["b", "a"])

    x = ["a", np.nan, "c", "c", "b", "a", "d"]
    out = utils.categorical_order(x)
    nt.assert_equal(out, ["a", "c", "b", "d"])

if LooseVersion(pd.__version__) >= "0.15":

    def check_load_dataset(name):
        ds = load_dataset(name, cache=False)
        assert(isinstance(ds, pd.DataFrame))

    def check_load_cached_dataset(name):
        # Test the cacheing using a temporary file.
        # With Python 3.2+, we could use the tempfile.TemporaryDirectory()
        # context manager instead of this try...finally statement
        tmpdir = tempfile.mkdtemp()
        try:
            # download and cache
            ds = load_dataset(name, cache=True, data_home=tmpdir)

            # use cached version
            ds2 = load_dataset(name, cache=True, data_home=tmpdir)
            pdt.assert_frame_equal(ds, ds2)

        finally:
            shutil.rmtree(tmpdir)

    @network(url="https://github.com/mwaskom/seaborn-data")
    def test_get_dataset_names():
        if not BeautifulSoup:
            raise nose.SkipTest("No BeautifulSoup available for parsing html")
        names = get_dataset_names()
        assert(len(names) > 0)
        assert(u"titanic" in names)

    @network(url="https://github.com/mwaskom/seaborn-data")
    def test_load_datasets():
        if not BeautifulSoup:
            raise nose.SkipTest("No BeautifulSoup available for parsing html")

        # Heavy test to verify that we can load all available datasets
        for name in get_dataset_names():
            # unfortunately @network somehow obscures this generator so it
            # does not get in effect, so we need to call explicitly
            # yield check_load_dataset, name
            check_load_dataset(name)

    @network(url="https://github.com/mwaskom/seaborn-data")
    def test_load_cached_datasets():
        if not BeautifulSoup:
            raise nose.SkipTest("No BeautifulSoup available for parsing html")

        # Heavy test to verify that we can load all available datasets
        for name in get_dataset_names():
            # unfortunately @network somehow obscures this generator so it
            # does not get in effect, so we need to call explicitly
            # yield check_load_dataset, name
            check_load_cached_dataset(name)


Machine Learning A-Z Template Folder/Part 2 - Regression/Section 8 - Decision Tree Regression/regression_template.py

# Regression Template

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Position_Salaries.csv')
X = dataset.iloc[:, 1:2].values
y = dataset.iloc[:, 2].values

# Splitting the dataset into the Training set and Test set
"""from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)"""

# Feature Scaling
"""from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)
sc_y = StandardScaler()
y_train = sc_y.fit_transform(y_train)"""

# Fitting the Regression Model to the dataset
# Create your regressor here

# Predicting a new result
y_pred = regressor.predict(6.5)

# Visualising the Regression results
plt.scatter(X, y, color = 'red')
plt.plot(X, regressor.predict(X), color = 'blue')
plt.title('Truth or Bluff (Regression Model)')
plt.xlabel('Position level')
plt.ylabel('Salary')
plt.show()

# Visualising the Regression results (for higher resolution and smoother curve)
X_grid = np.arange(min(X), max(X), 0.1)
X_grid = X_grid.reshape((len(X_grid), 1))
plt.scatter(X, y, color = 'red')
plt.plot(X_grid, regressor.predict(X_grid), color = 'blue')
plt.title('Truth or Bluff (Regression Model)')
plt.xlabel('Position level')
plt.ylabel('Salary')
plt.show()


import intern
from intern.remote.dvid import DVIDRemote
from intern.resource.dvid.resource import DataInstanceResource
from intern.resource.dvid.resource import RepositoryResource
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

########### NOTE ###########
# This test requires an accessible DVID instance

# DVID Data fetch:
dvid = DVIDRemote({"protocol": "http", "host": "localhost:8001",})

DATA_INSTANCE = "ex_EM"
ALIAS = "Test_alias"

########### Test Project API ###########
## Create DataInstanceResource and force the creation of a RepositoryResource
instance_setup_em = DataInstanceResource(
    DATA_INSTANCE, None, "uint8blk", ALIAS, "Example channel.", datatype="uint8"
)

# Get the channel and create a project
instance_actual_repo = dvid.create_project(instance_setup_em)
print("Repo UUID:" + instance_actual_repo)

# Create an instance within given repo(UUID)
instance_setup_anno = DataInstanceResource(
    DATA_INSTANCE + "_the_second",
    instance_actual_repo,
    "uint8blk",
    ALIAS,
    "Example channel.",
    datatype="uint8",
)

instance_actual_anno_uuid = dvid.create_project(instance_setup_anno)
print("Data Instance UUID: {}".format(instance_actual_anno_uuid))

# Create a dummy repo with the Repository Resource for deletion
instance_setup_em_delete = RepositoryResource(None, "Test_for_deletion")
instance_actual_em_delete_uuid = dvid.create_project(instance_setup_em_delete)
instance_actual_em_delete = dvid.delete_project(instance_setup_em_delete)
print("Successfully deleted Repo project: {}".format(instance_actual_em_delete_uuid))

# Delete the data instance of a repo
instance_setup_em_delete = DataInstanceResource(
    DATA_INSTANCE, None, "uint8blk", ALIAS, "Example channel.", datatype="uint8"
)
instance_actual_em_delete_uuid = dvid.create_project(instance_setup_em_delete)
dvid.delete_project(dvid.get_instance(instance_actual_em_delete_uuid, DATA_INSTANCE))
print(
    "Successfully deleted data instance project: {}".format(
        instance_actual_em_delete_uuid
    )
)

########### Test Versioning API ###########
# Set up a new project with a channel

instance_setup_merge = DataInstanceResource(
    DATA_INSTANCE + "_the_second",
    None,
    "uint8blk",
    "Mege_repo",
    "Example channel.",
    datatype="uint8",
)

chan_actual_parent1 = dvid.create_project(instance_setup_merge)
print("\nParent1 UUID: " + chan_actual_parent1)
commit_1 = dvid.commit(chan_actual_parent1, note="Test the commit")
branch_1 = dvid.branch(chan_actual_parent1, note="Test the versioning system once")

branch_2 = dvid.branch(chan_actual_parent1, note="Test the versioning system twice")

print("Created branches {} and {} from Parent1".format(branch_1, branch_2))

########### Test Metadat API ###########
# Set up a new project with a channel
print(dvid.get_info(instance_setup_merge))
print(dvid.get_server_info())
print(dvid.get_server_compiled_types())
dvid.server_reload_metadata()

########### Test Voluming API ###########
#

# Prepare the data
img = Image.open("<somedir>/*.png")
data_tile = np.asarray(img)
print(data_tile.shape)
data_tile = np.expand_dims(data_tile, axis=0)
data_tile = data_tile.copy(order="C")

# Create the project
instance_setup_up = DataInstanceResource(
    DATA_INSTANCE + "_the_second",
    None,
    "imagetile",
    "Upload Test",
    "Example channel.",
    datatype="uint8",
)
chan_actual_up = dvid.create_project(instance_setup_up)

# Create the cutout
dvid.create_cutout(instance_setup_up, 0, [0, 454], [0, 480], [0, 1], data_tile)
print("Create cutout successful")

# Get the cutout
got_cutout = dvid.get_cutout(instance_setup_up, 0, [0, 454], [0, 480], [0, 1])

# Check for equality
if (got_cutout == data_tile).all():
    print("Both tiles equate")


import ast
from unittest import TestCase
from textwrap import dedent

import pandas as pd
import numpy as np
from numpy.testing import assert_almost_equal
import nose.tools as nt

from asttools import (
    ast_equal
)

from ..manifest import (
    Expression,
    Manifest,
    _manifest
)

from ..exec_context import (
    ContextObject,
    SourceObject,
    ExecutionContext,
    get_source_key
)

from .common import ArangeSource

def grab_expression_from_assign(code):
    node = code.body[0].value
    expr = ast.Expression(lineno=0, col_offset=0, body=node)
    return expr

class TestExpression(TestCase):
    def test_expression(self):
        source = """
        arr = np.arange(20)
        res = np.sum(arr)
        """
        source = dedent(source)
        lines = source.strip().split('\n')
        load_names = [['np'], ['np', 'arr']]
        for i, line in enumerate(lines):
            code = ast.parse(line, '<>', 'exec')

            # expression must be evaluable, assignments are not
            with nt.assert_raises(Exception):
                Expression(code.body[0])

            extracted_expr = grab_expression_from_assign(code)
            # skip the assign
            base_expr = ast.parse(line.split('=')[1].strip(), mode='eval')
            exp1 = Expression(extracted_expr)
            exp2 = Expression(base_expr)
            nt.assert_equal(exp1, exp2)
            nt.assert_is_not(exp1, exp2)
            nt.assert_count_equal(exp1.load_names(), load_names[i])

    def test_single_line(self):
        """ Expressoins can only be single line """
        source = """
        np.arange(20)
        np.sum(arr)
        """
        source = dedent(source)
        code = ast.parse(source)

        # expression must be single line
        with nt.assert_raises(Exception):
            Expression(code)

        # single line still works
        Expression(code.body[0])
        Expression(code.body[1])

    def test_expression_conversion(self):
        """
        So I'm not 100% sure on converting all code into ast.Expressions.
        Right now it is what I'm doing, so might as well explicitly test?
        """
        source = """
        np.arange(20)
        np.sum(arr)
        """
        source = dedent(source)
        code = ast.parse(source)

        expr1 = Expression(code.body[0])
        nt.assert_is_instance(expr1.code, ast.Expression)
        expr2 = Expression(code.body[1])
        nt.assert_is_instance(expr2.code, ast.Expression)

        expr3 = Expression("np.arange(15)")
        nt.assert_is_instance(expr3.code, ast.Expression)

    def test_key(self):
        """ stable hash key """
        source = """
        np.arange(20)
        np.sum(arr)
        """
        source = dedent(source)
        code = ast.parse(source)

        expr1 = Expression(code.body[0])
        expr2 = Expression(code.body[1])

        import binascii
        # changed key to return str, same hash just different rep
        correct1 = b'}\xff\x1c\x0er\xe8k3\x84\x96R\x98\x9a\xa4\xe0i'
        correct1 = binascii.b2a_hex(correct1).decode('utf-8')
        correct2 = b'\xd6\x88\x08\xa2\xd0\x01\xa4\xc6\xabb\x1aTj\xce\x98\x18'
        correct2 = binascii.b2a_hex(correct2).decode('utf-8')
        # keys are stable and should not change between lifecycles
        nt.assert_equal(expr1.key, correct1)
        nt.assert_equal(expr2.key, correct2)

        # key also works for equals
        nt.assert_equal(expr1, correct1)
        nt.assert_equal(expr2, correct2)

    def test_copy(self):
        """
        test copy
        """
        source = """
        np.arange(20)
        """
        source = dedent(source)
        code = ast.parse(source)

        expr1 = Expression(code.body[0])
        expr2 = expr1.copy()
        # equivalent value
        nt.assert_true(ast_equal(expr1.code, expr2.code))
        # but not the same
        nt.assert_is_not(expr1.code, expr2.code)
        nt.assert_is_not(expr1.code.body, expr2.code.body)

        # mutability
        nt.assert_false(expr2.mutable)

        expr3 = expr1.copy(mutable=True)
        nt.assert_true(expr3.mutable)

    def test_mutability(self):
        """ test immutability """
        source = """
        np.arange(20)
        """
        source = dedent(source)
        code = ast.parse(source)

        new_num = ast.Num(n=3)

        expr1 = Expression(code.body[0])
        with nt.assert_raises_regexp(Exception, "This expression is not mutable"):
            expr1.replace(new_num, expr1.code.body, 'args', 0)

        expr2 = expr1.copy(mutable=True)
        old_key = expr2.key
        expr2.replace(new_num, expr2.code.body, 'args', 0)
        nt.assert_not_equal(expr2.key, old_key)

        # expr2 was changed
        nt.assert_false(ast_equal(expr1.code, expr2.code))
        nt.assert_equal(expr2.get_source(), 'np.arange(3)')

class TestManifest(TestCase):
    def test_eval(self):
        source = "d * string_test"

        context = {
            'd': 13,
            'string_test': 'string_test'
        }

        expr = Expression(source)
        exec_context = ExecutionContext(context)
        manifest = Manifest(expr, exec_context)
        nt.assert_equal(manifest.eval(), 'string_test' * 13)

    def test_equals(self):
        source = "d * string_test"

        context = {
            'd': 13,
            'string_test': 'string_test'
        }

        expr = Expression(source)
        exec_context = ExecutionContext(context)
        manifest = Manifest(expr, exec_context)
        manifest2 = Manifest(expr, exec_context)

        nt.assert_equal(manifest, manifest2)

        # change expression
        expr3 = Expression("d * string_test * 2")
        manifest3 = Manifest(expr3, exec_context)
        nt.assert_not_equal(manifest, manifest3)

        # change context
        context4 = {
            'd': 11,
            'string_test': 'string_test'
        }
        exec_context4 = ExecutionContext(context4)
        manifest4 = Manifest(expr, exec_context4)
        nt.assert_not_equal(manifest, manifest4)

    def test_nested_eval(self):
        """
        d * (1 + arr + arr2[10:])
        which is really two manifests

        arr_manifest = (1 + arr + arr2[10:])
        manfiest = (d * (arr_manifest))
        """

        arr_source = "1 + arr + arr2[10:]"
        aranger = ArangeSource()

        arr_context = {
            'arr': SourceObject(aranger, 10),
            'arr2': SourceObject(aranger, 20),
        }
        arr_expr = Expression(arr_source)
        arr_exec_context = ExecutionContext.from_ns(arr_context)
        arr_manifest = Manifest(arr_expr, arr_exec_context)

        source = "d * arr"
        context = {
            'd': 13,
            'arr': arr_manifest
        }
        expr = Expression(source)
        exec_context = ExecutionContext.from_ns(context)
        manifest = Manifest(expr, exec_context)

        correct = 13 * (1 + np.arange(10) + np.arange(20)[10:])

        # up till this point, everything is lazy
        nt.assert_equal(len(aranger.cache), 0)
        assert_almost_equal(correct, manifest.eval())
        nt.assert_equal(len(aranger.cache), 2)

    def test_hashable(self):
        source = "d * string_test"

        context = {
            'd': 13,
            'string_test': 'string_test'
        }

        expr = Expression(source)
        exec_context = ExecutionContext(context)
        manifest = Manifest(expr, exec_context)

        d = {}
        d[manifest] = manifest #hashable
        key = tuple([manifest.expression, manifest.context])
        # test key
        nt.assert_in(key, d)

        # a feature is being able to check expression.key for cases
        # where we don't have the source and just the stable key
        stable_key = tuple([expr.key, manifest.context])
        nt.assert_in(stable_key, d)

    def test_stateless(self):
        """
        stateless-ness of Manifest depends on context
        """
        source = "d * string_test"

        context = {
            'd': 13,
            'string_test': 'string_test'
        }

        expr = Expression(source)
        exec_context = ExecutionContext(context)
        manifest = Manifest(expr, exec_context)

        nt.assert_equal(manifest.stateless, True)

        context = {
            'd': 13,
            'string_test': object(),
        }

        expr = Expression(source)
        exec_context = ExecutionContext.from_ns(context)
        manifest = Manifest(expr, exec_context)
        nt.assert_equal(manifest.stateless, False)

def test_fragment():
    """
    This is a failing test atm. What I want is the ability to take two manifest
    and see whether one is within the other.

    A couple of notes. They sub-expression itself would obviously need to
    match. With each sub expression, you can have a subset of execution
    contexts. it is that subset that needs to match.

    Manifest 1:
        Expression:
            arr1 + np.log(arr2)
        ExecutionContext:
            arr1 = np.random(10)
            arr2 = np.arange(10)

    Manifest 2:
        Expression:
            np.log(arr1)
        ExecutionContext:
            arr1 = np.arange(10)

    Here manifest 2 should be considered subset of Manfiest 1, provided
    that np.arange is wrapped to be stateless.

    Now, currently our hash is done via the string repr. Since `arr2` in
    Manifest 1 is `arr1` in Manfiest 2, we currently wouldn't match.

    So we'd need to match the load name by value and not by name. I suppose
    one could have a modified ast_source that replaced load names with pos
    IDs.
    """
    c = 1
    df = pd.DataFrame(np.random.randn(30, 3), columns=['a', 'bob', 'c'])
    source = """pd.core.window.Rolling(np.log(df + 10), 5, min_periods=c).sum()"""
    ns = locals()
    ns.update({k:v for k, v in globals().items() if k not in ns})
    manifest = _manifest(source, ns)

    sub_mf = _manifest("np.log(df+10)", ns.copy())
    nt.assert_in(sub_mf, manifest)

    # new dataframe, does effect contains
    ns['df'] = pd.DataFrame(np.random.randn(30, 3), columns=['a', 'bob', 'c'])
    sub_mf = _manifest("np.log(df+10)", ns.copy())
    nt.assert_not_in(sub_mf, manifest)

    # c is changed but not part of fragment, so doesn't effect contains
    ns['c'] = 3
    manifest = _manifest(source, ns)
    sub_mf = _manifest("np.log(df+10)", ns.copy())
    nt.assert_in(sub_mf, manifest)

def test_fragment_var_name():
    """
    This should match even though the variable names are different.
    """
    c = 1
    df = pd.DataFrame(np.random.randn(30, 3), columns=['a', 'bob', 'c'])
    source = """pd.core.window.Rolling(np.log(df + 10), 5, min_periods=c).sum()"""
    ns = locals()
    ns.update({k:v for k, v in globals().items() if k not in ns})
    manifest = _manifest(source, ns)

    # use blah instead of df. same code.
    ns['blah'] = ns['df']
    sub_mf = _manifest("np.log(blah+10)", ns)
    nt.assert_in(sub_mf, manifest)

    # now change blah to be a differnt value
    ns['blah'] = 1
    sub_mf = _manifest("np.log(blah+10)", ns)
    nt.assert_not_in(sub_mf, manifest)

def test_fragment_order_of_ops():
    """
    So, in a pure math sense, you should be able to
    do this replacement:

    E1 = a + b + a + (a + b)
    S = b + a + (a + b)
    E3 = a + S
    E1 == E3

    But since in python, the order of operations matters, you can't just
    treat that as a subset.(a + b) is not always the same as (b + a)

    Dumb example:

    class Bob:
        def __add__(self, other):
            return other

    a = Bob()
    b = Bob()
    nt.assert_not_equal(a + b, b + a)
    """
    # TODO, is there a way to subset when dealing with types where operations
    # are commutative?
    ns = {'a': 1, 'b': 2}
    manifest = _manifest("a + b + a + (a + b)", ns)

    manifest2 = _manifest("b + a + (a + b)", ns)
    nt.assert_not_in(manifest2, manifest)

def test_manifest_partial():
    """
    Mechanism where the take a Manifest and supply a partial value via
    another Manifest.
    """
    ns = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
    parent = _manifest("a + (c + d)", ns)

    sub = _manifest("(x + y)", {'x': 3, 'y': 4})

    # note we are purposely giving wrong answer
    items = {sub: 3}
    test = parent.eval_with(items, ignore_var_names=True)
    nt.assert_equal(test, 4)

    # parent unaffected
    nt.assert_equal(parent.eval(), 8)

    # sub also un affected
    nt.assert_equal(sub.eval(), 7)

def test_manifest_partial_multi():
    ns = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
    parent = _manifest("a + (c + d) + (a + b)", ns)

    # we are expecting these to match by execution context
    sub = _manifest("(x + y)", {'x': 3, 'y': 4})
    sub2 = _manifest("(x + y)", {'x': 1, 'y': 2})

    items = {sub: sub.eval(), sub2: sub2.eval()}

    # this errors since we don't multi match on the ast_contains
    test = parent.eval_with(items, ignore_var_names=True)
    nt.assert_equal(test, 11)

    items = {sub: sub, sub2: sub2}

    test = parent.eval_with(items, ignore_var_names=True)
    nt.assert_equal(test, 11)

    # pass in only manifest
    test = parent.eval_with([sub, sub2], ignore_var_names=True)
    nt.assert_equal(test, 11)

def test_eval_with_execution_count():
    class Value:
        """ value that keeps track of when it is used in ops """
        def __init__(self, value):
            self.value = value
            self.op_count = 0

        def get_obj(self):
            return self.value

        def __add__(self, other):
            self.op_count += 1
            return self.value + other

        def __radd__(self, other):
            self.op_count += 1
            return self.value + other

    ns = {
        'a': Value(1),
        'b': Value(2),
        'c': Value(3),
        'd': Value(4),
        'e': Value(5)
    }
    parent = _manifest("e + (c + d) + (a + b)", ns)

    # we are expecting these to match by execution context
    sub = _manifest("(a + b)", ns)
    sub2 = _manifest("(c + d)", ns)

    items = {sub: sub.eval(), sub2: sub2.eval()}

    # a through d should have been used
    nt.assert_equal(ns['a'].op_count, 1)
    nt.assert_equal(ns['b'].op_count, 1)
    nt.assert_equal(ns['c'].op_count, 1)
    nt.assert_equal(ns['d'].op_count, 1)
    nt.assert_equal(ns['e'].op_count, 0)

    res = parent.eval_with(items)
    nt.assert_equal(res, 1+2+3+4+5)

    # make sure we did not use the Value again
    nt.assert_equal(ns['a'].op_count, 1)
    nt.assert_equal(ns['b'].op_count, 1)
    nt.assert_equal(ns['c'].op_count, 1)
    nt.assert_equal(ns['d'].op_count, 1)
    nt.assert_equal(ns['e'].op_count, 1) # gets used

    # normal non partial eval
    res = parent.eval()
    nt.assert_equal(res, 1+2+3+4+5)

    # since we did a full eval, everything got run again
    nt.assert_equal(ns['a'].op_count, 2)
    nt.assert_equal(ns['b'].op_count, 2)
    nt.assert_equal(ns['c'].op_count, 2)
    nt.assert_equal(ns['d'].op_count, 2)
    nt.assert_equal(ns['e'].op_count, 2) # gets used

def test_expanded_multi_nested_partial():
    # we are expecting these to match by execution context
    ns = {'test1':0, 'test2': 1}
    leaf = _manifest("(test1 + test2)", ns)

    ns = {'x':1, 'y': leaf}
    xy = _manifest("(x + y)", ns)

    ns = {'a': 1, 'b': xy}
    sub = _manifest("(a + b)", ns)

    parent_ns = {'e': 3, 'a': sub}
    parent = _manifest("e + a", parent_ns)

    expanded = parent.expand()
    nt.assert_count_equal(expanded.context.keys(),
                          ['a', 'e', 'x', 'test1', 'test2'])
    nt.assert_equal(expanded.expression.get_source(),
                    "(e + (a + (x + (test1 + test2))))")
    nt.assert_equal(expanded.eval(), 6)


'''
Splits Complete Genomics masterVar files into chromosome specific masterVar
files when given an input file path and an output directory path.

e.g. >python masterVar_chr_split.py -i /path/to/masterVar.tsv.bz2 -o /path/to/output_dir/

Python package dependencies:
pandas, numpy
python 2.7 for argparse module
'''

import pandas as pd
import os, sys
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-i', type=str,help='Specifies the input file, /path/to/CG_data/masterVar.tsv.bz2')
parser.add_argument('-o', type=str,help='Specifies the output directory, e.g. /path/to/CG_data/chromosome/')

def chr_split_mastervar(f_path, target_path):

    #Get header for masterVar
    header = os.popen('bzcat ' + f_path+ ' | head -100 | grep chromosome -n').readlines()

    #Creating Reader object for iterating through NA12878 CG masterVar file
    skip_rows = int(header[0].split(":")[0]) -1
    mastervar_headings = os.popen('head -' + str(skip_rows) + f_path).readlines()

    #Creating pandas dataframe with chunksize 200,000 lines
    chunk = pd.read_table(f_path, chunksize=200000, sep="\t", skiprows=skip_rows,compression='bz2',dtype=object)
    chunk.columns = header[0].rstrip('\n').split(":")[1].split("\t")  #relabeling columns

    prev_chr = 'chr1'  #tracking chromosome position
    prev_target_write_file = None
    for mastervar in chunk:  #iterate through mastervar file
        for current_chrom,chr_df in mastervar.groupby(['chromosome']):  #split dataframe by chromosome for writing

            #check for increment to new chromosome
            if prev_chr != current_chrom:
                os.system('bzip2 ' + prev_target_write_file) #compress last chromosome file
                prev_chr = current_chrom

            #specifying output file path and chromosome-specific name
            file_name = f_path.split("/")[-1].rstrip(".tsv.bz2") #getting file prefix
            target_path = target_path.rstrip("/")+"/"  #ensuring target path ends with fwd slash
            write_target_file_path = target_path +file_name + "_" + current_chrom +".tsv"  #specify target directory and chrom file name

            #print write_target_file_path
            if len(os.popen('find '+ write_target_file_path + '').readlines()) == 0:  #checking for output file
                os.system('bzcat '+ f_path + '| head -' + str(skip_rows) + " > " +write_target_file_path) #writing header if no output file found
                chr_df.to_csv(write_target_file_path, sep="\t", index=False, mode='a')  #writing chromosome specific variants to output file
            else:  #Suppress header if target file found
                chr_df.to_csv(write_target_file_path, sep="\t", index=False, mode='a', header=False)  #writing chromosome specifc variants to output file w/o header

            prev_target_write_file = write_target_file_path  #increment to current write_target_file_path

    return 'complete'

opts = parser.parse_known_args()
f_path, target_path = opts[0].i, opts[0].o
assert f_path.split(".")[-2:] == ['tsv','bz2'], "expecting masterVar input file suffix .tsv.bz2"

test = chr_split_mastervar(f_path, target_path)
if test == 'complete':
    print 'All chromosomes processed'


"""This module is designed for community supported date conversion functions"""
from pandas.compat import range, map
import numpy as np
import pandas._libs.lib as lib

def parse_date_time(date_col, time_col):
    date_col = _maybe_cast(date_col)
    time_col = _maybe_cast(time_col)
    return lib.try_parse_date_and_time(date_col, time_col)

def parse_date_fields(year_col, month_col, day_col):
    year_col = _maybe_cast(year_col)
    month_col = _maybe_cast(month_col)
    day_col = _maybe_cast(day_col)
    return lib.try_parse_year_month_day(year_col, month_col, day_col)

def parse_all_fields(year_col, month_col, day_col, hour_col, minute_col,
                     second_col):
    year_col = _maybe_cast(year_col)
    month_col = _maybe_cast(month_col)
    day_col = _maybe_cast(day_col)
    hour_col = _maybe_cast(hour_col)
    minute_col = _maybe_cast(minute_col)
    second_col = _maybe_cast(second_col)
    return lib.try_parse_datetime_components(year_col, month_col, day_col,
                                             hour_col, minute_col, second_col)

def generic_parser(parse_func, *cols):
    N = _check_columns(cols)
    results = np.empty(N, dtype=object)

    for i in range(N):
        args = [c[i] for c in cols]
        results[i] = parse_func(*args)

    return results

def _maybe_cast(arr):
    if not arr.dtype.type == np.object_:
        arr = np.array(arr, dtype=object)
    return arr

def _check_columns(cols):
    if not len(cols):
        raise AssertionError("There must be at least 1 column")

    head, tail = cols[0], cols[1:]

    N = len(head)

    for i, n in enumerate(map(len, tail)):
        if n != N:
            raise AssertionError('All columns must have the same length: {0}; '
                                 'column {1} has length {2}'.format(N, i, n))

    return N


#
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from copy import copy
import warnings

import pytz
import pandas as pd
import numpy as np

from datetime import datetime

from itertools import groupby, chain
from six.moves import filter
from six import (
    exec_,
    iteritems,
    itervalues,
    string_types,
)
from operator import attrgetter

from zipline.errors import (
    AddTermPostInit,
    OrderDuringInitialize,
    OverrideCommissionPostInit,
    OverrideSlippagePostInit,
    RegisterAccountControlPostInit,
    RegisterTradingControlPostInit,
    UnsupportedCommissionModel,
    UnsupportedOrderParameters,
    UnsupportedSlippageModel,
)
from zipline.finance.trading import TradingEnvironment
from zipline.finance.blotter import Blotter
from zipline.finance.commission import PerShare, PerTrade, PerDollar
from zipline.finance.controls import (
    LongOnly,
    MaxOrderCount,
    MaxOrderSize,
    MaxPositionSize,
    MaxLeverage,
    RestrictedListOrder
)
from zipline.finance.execution import (
    LimitOrder,
    MarketOrder,
    StopLimitOrder,
    StopOrder,
)
from zipline.finance.performance import PerformanceTracker
from zipline.finance.slippage import (
    VolumeShareSlippage,
    SlippageModel,
    transact_partial
)
from zipline.assets import Asset, Future
from zipline.assets.futures import FutureChain
from zipline.gens.composites import date_sorted_sources
from zipline.gens.tradesimulation import AlgorithmSimulator
from zipline.modelling.engine import (
    NoOpFFCEngine,
    SimpleFFCEngine,
)
from zipline.sources import DataFrameSource, DataPanelSource
from zipline.utils.api_support import (
    api_method,
    require_not_initialized,
    ZiplineAPI,
)
import zipline.utils.events
from zipline.utils.events import (
    EventManager,
    make_eventrule,
    DateRuleFactory,
    TimeRuleFactory,
)
from zipline.utils.factory import create_simulation_parameters
from zipline.utils.math_utils import tolerant_equals

import zipline.protocol
from zipline.protocol import Event

from zipline.history import HistorySpec
from zipline.history.history_container import HistoryContainer

DEFAULT_CAPITAL_BASE = float("1.0e5")

class TradingAlgorithm(object):
    """
    Base class for trading algorithms. Inherit and overload
    initialize() and handle_data(data).

    A new algorithm could look like this:
    ```
    from zipline.api import order, symbol

    def initialize(context):
        context.sid = symbol('AAPL')
        context.amount = 100

    def handle_data(context, data):
        sid = context.sid
        amount = context.amount
        order(sid, amount)
    ```
    To then to run this algorithm pass these functions to
    TradingAlgorithm:

    my_algo = TradingAlgorithm(initialize, handle_data)
    stats = my_algo.run(data)

    """

    def __init__(self, *args, **kwargs):
        """Initialize sids and other state variables.

        :Arguments:
        :Optional:
            initialize : function
                Function that is called with a single
                argument at the begninning of the simulation.
            handle_data : function
                Function that is called with 2 arguments
                (context and data) on every bar.
            script : str
                Algoscript that contains initialize and
                handle_data function definition.
            data_frequency : {'daily', 'minute'}
               The duration of the bars.
            capital_base : float <default: 1.0e5>
               How much capital to start with.
            instant_fill : bool <default: False>
               Whether to fill orders immediately or on next bar.
            asset_finder : An AssetFinder object
                A new AssetFinder object to be used in this TradingEnvironment
            asset_metadata: can be either:
                            - dict
                            - pandas.DataFrame
                            - object with 'read' property
                If dict is provided, it must have the following structure:
                * keys are the identifiers
                * values are dicts containing the metadata, with the metadata
                  field name as the key
                If pandas.DataFrame is provided, it must have the
                following structure:
                * column names must be the metadata fields
                * index must be the different asset identifiers
                * array contents should be the metadata value
                If an object with a 'read' property is provided, 'read' must
                return rows containing at least one of 'sid' or 'symbol' along
                with the other metadata fields.
            identifiers : List
                Any asset identifiers that are not provided in the
                asset_metadata, but will be traded by this TradingAlgorithm
        """
        self.sources = []

        # List of trading controls to be used to validate orders.
        self.trading_controls = []

        # List of account controls to be checked on each bar.
        self.account_controls = []

        self._recorded_vars = {}
        self.namespace = kwargs.get('namespace', {})

        self._platform = kwargs.pop('platform', 'zipline')

        self.logger = None

        self.benchmark_return_source = None

        # default components for transact
        self.slippage = VolumeShareSlippage()
        self.commission = PerShare()

        self.instant_fill = kwargs.pop('instant_fill', False)

        # set the capital base
        self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE)

        self.sim_params = kwargs.pop('sim_params', None)
        if self.sim_params is None:
            self.sim_params = create_simulation_parameters(
                capital_base=self.capital_base,
                start=kwargs.pop('start', None),
                end=kwargs.pop('end', None)
            )
        self.perf_tracker = PerformanceTracker(self.sim_params)

        # Update the TradingEnvironment with the provided asset metadata
        self.trading_environment = kwargs.pop('env',
                                              TradingEnvironment.instance())
        self.trading_environment.update_asset_finder(
            asset_finder=kwargs.pop('asset_finder', None),
            asset_metadata=kwargs.pop('asset_metadata', None),
            identifiers=kwargs.pop('identifiers', None)
        )
        # Pull in the environment's new AssetFinder for quick reference
        self.asset_finder = self.trading_environment.asset_finder
        self.init_engine(kwargs.pop('ffc_loader', None))

        # Maps from name to Term
        self._filters = {}
        self._factors = {}
        self._classifiers = {}

        self.blotter = kwargs.pop('blotter', None)
        if not self.blotter:
            self.blotter = Blotter()

        # Set the dt initally to the period start by forcing it to change
        self.on_dt_changed(self.sim_params.period_start)

        self.portfolio_needs_update = True
        self.account_needs_update = True
        self.performance_needs_update = True
        self._portfolio = None
        self._account = None

        self.history_container_class = kwargs.pop(
            'history_container_class', HistoryContainer,
        )
        self.history_container = None
        self.history_specs = {}

        # If string is passed in, execute and get reference to
        # functions.
        self.algoscript = kwargs.pop('script', None)

        self._initialize = None
        self._before_trading_start = None
        self._analyze = None

        self.event_manager = EventManager()

        if self.algoscript is not None:
            filename = kwargs.pop('algo_filename', None)
            if filename is None:
                filename = '<string>'
            code = compile(self.algoscript, filename, 'exec')
            exec_(code, self.namespace)
            self._initialize = self.namespace.get('initialize')
            if 'handle_data' not in self.namespace:
                raise ValueError('You must define a handle_data function.')
            else:
                self._handle_data = self.namespace['handle_data']

            self._before_trading_start = \
                self.namespace.get('before_trading_start')
            # Optional analyze function, gets called after run
            self._analyze = self.namespace.get('analyze')

        elif kwargs.get('initialize') and kwargs.get('handle_data'):
            if self.algoscript is not None:
                raise ValueError('You can not set script and \
                initialize/handle_data.')
            self._initialize = kwargs.pop('initialize')
            self._handle_data = kwargs.pop('handle_data')
            self._before_trading_start = kwargs.pop('before_trading_start',
                                                    None)

        self.event_manager.add_event(
            zipline.utils.events.Event(
                zipline.utils.events.Always(),
                # We pass handle_data.__func__ to get the unbound method.
                # We will explicitly pass the algorithm to bind it again.
                self.handle_data.__func__,
            ),
            prepend=True,
        )

        # If method not defined, NOOP
        if self._initialize is None:
            self._initialize = lambda x: None

        # Alternative way of setting data_frequency for backwards
        # compatibility.
        if 'data_frequency' in kwargs:
            self.data_frequency = kwargs.pop('data_frequency')

        self._most_recent_data = None

        # Prepare the algo for initialization
        self.initialized = False
        self.initialize_args = args
        self.initialize_kwargs = kwargs

    def init_engine(self, loader):
        """
        Construct and save an FFCEngine from loader.

        If loader is None, constructs a NoOpFFCEngine.
        """
        if loader is not None:
            self.engine = SimpleFFCEngine(
                loader,
                self.trading_environment.trading_days,
                self.asset_finder,
            )
        else:
            self.engine = NoOpFFCEngine()

    def initialize(self, *args, **kwargs):
        """
        Call self._initialize with `self` made available to Zipline API
        functions.
        """
        with ZiplineAPI(self):
            self._initialize(self)

    def before_trading_start(self, data):
        if self._before_trading_start is None:
            return

        self._before_trading_start(self, data)

    def handle_data(self, data):
        self._most_recent_data = data
        if self.history_container:
            self.history_container.update(data, self.datetime)

        self._handle_data(self, data)

        # Unlike trading controls which remain constant unless placing an
        # order, account controls can change each bar. Thus, must check
        # every bar no matter if the algorithm places an order or not.
        self.validate_account_controls()

    def analyze(self, perf):
        if self._analyze is None:
            return

        with ZiplineAPI(self):
            self._analyze(self, perf)

    def __repr__(self):
        """
        N.B. this does not yet represent a string that can be used
        to instantiate an exact copy of an algorithm.

        However, it is getting close, and provides some value as something
        that can be inspected interactively.
        """
        return """
{class_name}(
    capital_base={capital_base}
    sim_params={sim_params},
    initialized={initialized},
    slippage={slippage},
    commission={commission},
    blotter={blotter},
    recorded_vars={recorded_vars})
""".strip().format(class_name=self.__class__.__name__,
                   capital_base=self.capital_base,
                   sim_params=repr(self.sim_params),
                   initialized=self.initialized,
                   slippage=repr(self.slippage),
                   commission=repr(self.commission),
                   blotter=repr(self.blotter),
                   recorded_vars=repr(self.recorded_vars))

    def _create_data_generator(self, source_filter, sim_params=None):
        """
        Create a merged data generator using the sources attached to this
        algorithm.

        ::source_filter:: is a method that receives events in date
        sorted order, and returns True for those events that should be
        processed by the zipline, and False for those that should be
        skipped.
        """
        if sim_params is None:
            sim_params = self.sim_params

        if self.benchmark_return_source is None:
            if sim_params.data_frequency == 'minute' or \
               sim_params.emission_rate == 'minute':
                def update_time(date):
                    return self.trading_environment.get_open_and_close(date)[1]
            else:
                def update_time(date):
                    return date
            benchmark_return_source = [
                Event({'dt': update_time(dt),
                       'returns': ret,
                       'type': zipline.protocol.DATASOURCE_TYPE.BENCHMARK,
                       'source_id': 'benchmarks'})
                for dt, ret in
                self.trading_environment.benchmark_returns.iteritems()
                if dt.date() >= sim_params.period_start.date() and
                dt.date() <= sim_params.period_end.date()
            ]
        else:
            benchmark_return_source = self.benchmark_return_source

        date_sorted = date_sorted_sources(*self.sources)

        if source_filter:
            date_sorted = filter(source_filter, date_sorted)

        with_benchmarks = date_sorted_sources(benchmark_return_source,
                                              date_sorted)

        # Group together events with the same dt field. This depends on the
        # events already being sorted.
        return groupby(with_benchmarks, attrgetter('dt'))

    def _create_generator(self, sim_params, source_filter=None):
        """
        Create a basic generator setup using the sources to this algorithm.

        ::source_filter:: is a method that receives events in date
        sorted order, and returns True for those events that should be
        processed by the zipline, and False for those that should be
        skipped.
        """

        if not self.initialized:
            self.initialize(*self.initialize_args, **self.initialize_kwargs)
            self.initialized = True

        if self.perf_tracker is None:
            # HACK: When running with the `run` method, we set perf_tracker to
            # None so that it will be overwritten here.
            self.perf_tracker = PerformanceTracker(sim_params)

        self.portfolio_needs_update = True
        self.account_needs_update = True
        self.performance_needs_update = True

        self.data_gen = self._create_data_generator(source_filter, sim_params)

        self.trading_client = AlgorithmSimulator(self, sim_params)

        transact_method = transact_partial(self.slippage, self.commission)
        self.set_transact(transact_method)

        return self.trading_client.transform(self.data_gen)

    def get_generator(self):
        """
        Override this method to add new logic to the construction
        of the generator. Overrides can use the _create_generator
        method to get a standard construction generator.
        """
        return self._create_generator(self.sim_params)

    # TODO: make a new subclass, e.g. BatchAlgorithm, and move
    # the run method to the subclass, and refactor to put the
    # generator creation logic into get_generator.
    def run(self, source, overwrite_sim_params=True,
            benchmark_return_source=None):
        """Run the algorithm.

        :Arguments:
            source : can be either:
                     - pandas.DataFrame
                     - zipline source
                     - list of sources

               If pandas.DataFrame is provided, it must have the
               following structure:
               * column names must be the different asset identifiers
               * index must be DatetimeIndex
               * array contents should be price info.

        :Returns:
            daily_stats : pandas.DataFrame
              Daily performance metrics such as returns, alpha etc.

        """

        # Ensure that source is a DataSource object
        if isinstance(source, list):
            if overwrite_sim_params:
                warnings.warn("""List of sources passed, will not attempt to extract start and end
 dates. Make sure to set the correct fields in sim_params passed to
 __init__().""", UserWarning)
                overwrite_sim_params = False
        elif isinstance(source, pd.DataFrame):
            # if DataFrame provided, map columns to sids and wrap
            # in DataFrameSource
            copy_frame = source.copy()
            copy_frame.columns = \
                self.asset_finder.map_identifier_index_to_sids(
                    source.columns, source.index[0]
                )
            source = DataFrameSource(copy_frame)

        elif isinstance(source, pd.Panel):
            # If Panel provided, map items to sids and wrap
            # in DataPanelSource
            copy_panel = source.copy()
            copy_panel.items = self.asset_finder.map_identifier_index_to_sids(
                source.items, source.major_axis[0]
            )
            source = DataPanelSource(copy_panel)

        if isinstance(source, list):
            self.set_sources(source)
        else:
            self.set_sources([source])

        # Override sim_params if params are provided by the source.
        if overwrite_sim_params:
            if hasattr(source, 'start'):
                self.sim_params.period_start = source.start
            if hasattr(source, 'end'):
                self.sim_params.period_end = source.end
            # Changing period_start and period_close might require updating
            # of first_open and last_close.
            self.sim_params._update_internal()

        # The sids field of the source is the reference for the universe at
        # the start of the run
        self._current_universe = set()
        for source in self.sources:
            for sid in source.sids:
                self._current_universe.add(sid)
        # Check that all sids from the source are accounted for in
        # the AssetFinder. This retrieve call will raise an exception if the
        # sid is not found.
        for sid in self._current_universe:
            self.asset_finder.retrieve_asset(sid)

        # force a reset of the performance tracker, in case
        # this is a repeat run of the algorithm.
        self.perf_tracker = None

        # create zipline
        self.gen = self._create_generator(self.sim_params)

        # Create history containers
        if self.history_specs:
            self.history_container = self.history_container_class(
                self.history_specs,
                self.current_universe(),
                self.sim_params.first_open,
                self.sim_params.data_frequency,
            )

        # loop through simulated_trading, each iteration returns a
        # perf dictionary
        perfs = []
        for perf in self.gen:
            perfs.append(perf)

        # convert perf dict to pandas dataframe
        daily_stats = self._create_daily_stats(perfs)

        self.analyze(daily_stats)

        return daily_stats

    def _create_daily_stats(self, perfs):
        # create daily and cumulative stats dataframe
        daily_perfs = []
        # TODO: the loop here could overwrite expected properties
        # of daily_perf. Could potentially raise or log a
        # warning.
        for perf in perfs:
            if 'daily_perf' in perf:

                perf['daily_perf'].update(
                    perf['daily_perf'].pop('recorded_vars')
                )
                perf['daily_perf'].update(perf['cumulative_risk_metrics'])
                daily_perfs.append(perf['daily_perf'])
            else:
                self.risk_report = perf

        daily_dts = [np.datetime64(perf['period_close'], utc=True)
                     for perf in daily_perfs]
        daily_stats = pd.DataFrame(daily_perfs, index=daily_dts)

        return daily_stats

    @api_method
    def add_transform(self, transform, days=None):
        """
        Ensures that the history container will have enough size to service
        a simple transform.

        :Arguments:
            transform : string
                The transform to add. must be an element of:
                {'mavg', 'stddev', 'vwap', 'returns'}.
            days : int <default=None>
                The maximum amount of days you will want for this transform.
                This is not needed for 'returns'.
        """
        if transform not in {'mavg', 'stddev', 'vwap', 'returns'}:
            raise ValueError('Invalid transform')

        if transform == 'returns':
            if days is not None:
                raise ValueError('returns does use days')

            self.add_history(2, '1d', 'price')
            return
        elif days is None:
            raise ValueError('no number of days specified')

        if self.sim_params.data_frequency == 'daily':
            mult = 1
            freq = '1d'
        else:
            mult = 390
            freq = '1m'

        bars = mult * days
        self.add_history(bars, freq, 'price')

        if transform == 'vwap':
            self.add_history(bars, freq, 'volume')

    @api_method
    def get_environment(self, field='platform'):
        env = {
            'arena': self.sim_params.arena,
            'data_frequency': self.sim_params.data_frequency,
            'start': self.sim_params.first_open,
            'end': self.sim_params.last_close,
            'capital_base': self.sim_params.capital_base,
            'platform': self._platform
        }
        if field == '*':
            return env
        else:
            return env[field]

    def add_event(self, rule=None, callback=None):
        """
        Adds an event to the algorithm's EventManager.
        """
        self.event_manager.add_event(
            zipline.utils.events.Event(rule, callback),
        )

    @api_method
    def schedule_function(self,
                          func,
                          date_rule=None,
                          time_rule=None,
                          half_days=True):
        """
        Schedules a function to be called with some timed rules.
        """
        date_rule = date_rule or DateRuleFactory.every_day()
        time_rule = ((time_rule or TimeRuleFactory.market_open())
                     if self.sim_params.data_frequency == 'minute' else
                     # If we are in daily mode the time_rule is ignored.
                     zipline.utils.events.Always())

        self.add_event(
            make_eventrule(date_rule, time_rule, half_days),
            func,
        )

    @api_method
    def record(self, *args, **kwargs):
        """
        Track and record local variable (i.e. attributes) each day.
        """
        # Make 2 objects both referencing the same iterator
        args = [iter(args)] * 2

        # Zip generates list entries by calling `next` on each iterator it
        # receives.  In this case the two iterators are the same object, so the
        # call to next on args[0] will also advance args[1], resulting in zip
        # returning (a,b) (c,d) (e,f) rather than (a,a) (b,b) (c,c) etc.
        positionals = zip(*args)
        for name, value in chain(positionals, iteritems(kwargs)):
            self._recorded_vars[name] = value

    @api_method
    def symbol(self, symbol_str):
        """
        Default symbol lookup for any source that directly maps the
        symbol to the Asset (e.g. yahoo finance).
        """
        return self.asset_finder.lookup_symbol_resolve_multiple(
            symbol_str,
            as_of_date=self.sim_params.period_end
        )

    @api_method
    def symbols(self, *args):
        """
        Default symbols lookup for any source that directly maps the
        symbol to the Asset (e.g. yahoo finance).
        """
        return [self.symbol(identifier) for identifier in args]

    @api_method
    def sid(self, a_sid):
        """
        Default sid lookup for any source that directly maps the integer sid
        to the Asset.
        """
        return self.asset_finder.retrieve_asset(a_sid)

    @api_method
    def future_chain(self, root_symbol, as_of_date=None):
        """ Look up a future chain with the specified parameters.

        Parameters
        ----------
        root_symbol : str
            The root symbol of a future chain.
        as_of_date : datetime.datetime or pandas.Timestamp or str, optional
            Date at which the chain determination is rooted. I.e. the
            existing contract whose notice date is first after this date is
            the primary contract, etc.

        Returns
        -------
        FutureChain
            The future chain matching the specified parameters.

        Raises
        ------
        RootSymbolNotFound
            If a future chain could not be found for the given root symbol.
        """
        return FutureChain(
            asset_finder=self.asset_finder,
            get_datetime=self.get_datetime,
            root_symbol=root_symbol.upper(),
            as_of_date=as_of_date
        )

    def _calculate_order_value_amount(self, asset, value):
        """
        Calculates how many shares/contracts to order based on the type of
        asset being ordered.
        """
        last_price = self.trading_client.current_data[asset].price

        if tolerant_equals(last_price, 0):
            zero_message = "Price of 0 for {psid}; can't infer value".format(
                psid=asset
            )
            if self.logger:
                self.logger.debug(zero_message)
            # Don't place any order
            return 0

        if isinstance(asset, Future):
            value_multiplier = asset.contract_multiplier
        else:
            value_multiplier = 1

        return value / (last_price * value_multiplier)

    @api_method
    def order(self, sid, amount,
              limit_price=None,
              stop_price=None,
              style=None):
        """
        Place an order using the specified parameters.
        """

        def round_if_near_integer(a, epsilon=1e-4):
            """
            Round a to the nearest integer if that integer is within an epsilon
            of a.
            """
            if abs(a - round(a)) <= epsilon:
                return round(a)
            else:
                return a

        # Truncate to the integer share count that's either within .0001 of
        # amount or closer to zero.
        # E.g. 3.9999 -> 4.0; 5.5 -> 5.0; -5.5 -> -5.0
        amount = int(round_if_near_integer(amount))

        # Raises a ZiplineError if invalid parameters are detected.
        self.validate_order_params(sid,
                                   amount,
                                   limit_price,
                                   stop_price,
                                   style)

        # Convert deprecated limit_price and stop_price parameters to use
        # ExecutionStyle objects.
        style = self.__convert_order_params_for_blotter(limit_price,
                                                        stop_price,
                                                        style)
        return self.blotter.order(sid, amount, style)

    def validate_order_params(self,
                              asset,
                              amount,
                              limit_price,
                              stop_price,
                              style):
        """
        Helper method for validating parameters to the order API function.

        Raises an UnsupportedOrderParameters if invalid arguments are found.
        """

        if not self.initialized:
            raise OrderDuringInitialize(
                msg="order() can only be called from within handle_data()"
            )

        if style:
            if limit_price:
                raise UnsupportedOrderParameters(
                    msg="Passing both limit_price and style is not supported."
                )

            if stop_price:
                raise UnsupportedOrderParameters(
                    msg="Passing both stop_price and style is not supported."
                )

        if not isinstance(asset, Asset):
            raise UnsupportedOrderParameters(
                msg="Passing non-Asset argument to 'order()' is not supported."
                    " Use 'sid()' or 'symbol()' methods to look up an Asset."
            )

        for control in self.trading_controls:
            control.validate(asset,
                             amount,
                             self.updated_portfolio(),
                             self.get_datetime(),
                             self.trading_client.current_data)

    @staticmethod
    def __convert_order_params_for_blotter(limit_price, stop_price, style):
        """
        Helper method for converting deprecated limit_price and stop_price
        arguments into ExecutionStyle instances.

        This function assumes that either style == None or (limit_price,
        stop_price) == (None, None).
        """
        # TODO_SS: DeprecationWarning for usage of limit_price and stop_price.
        if style:
            assert (limit_price, stop_price) == (None, None)
            return style
        if limit_price and stop_price:
            return StopLimitOrder(limit_price, stop_price)
        if limit_price:
            return LimitOrder(limit_price)
        if stop_price:
            return StopOrder(stop_price)
        else:
            return MarketOrder()

    @api_method
    def order_value(self, sid, value,
                    limit_price=None, stop_price=None, style=None):
        """
        Place an order by desired value rather than desired number of shares.
        If the requested sid is found in the universe, the requested value is
        divided by its price to imply the number of shares to transact.
        If the Asset being ordered is a Future, the 'value' calculated
        is actually the exposure, as Futures have no 'value'.

        value > 0 :: Buy/Cover
        value < 0 :: Sell/Short
        Market order:    order(sid, value)
        Limit order:     order(sid, value, limit_price)
        Stop order:      order(sid, value, None, stop_price)
        StopLimit order: order(sid, value, limit_price, stop_price)
        """
        amount = self._calculate_order_value_amount(sid, value)
        return self.order(sid, amount,
                          limit_price=limit_price,
                          stop_price=stop_price,
                          style=style)

    @property
    def recorded_vars(self):
        return copy(self._recorded_vars)

    @property
    def portfolio(self):
        return self.updated_portfolio()

    def updated_portfolio(self):
        if self.portfolio_needs_update:
            self._portfolio = \
                self.perf_tracker.get_portfolio(self.performance_needs_update)
            self.portfolio_needs_update = False
            self.performance_needs_update = False
        return self._portfolio

    @property
    def account(self):
        return self.updated_account()

    def updated_account(self):
        if self.account_needs_update:
            self._account = \
                self.perf_tracker.get_account(self.performance_needs_update)
            self.account_needs_update = False
            self.performance_needs_update = False
        return self._account

    def set_logger(self, logger):
        self.logger = logger

    def on_dt_changed(self, dt):
        """
        Callback triggered by the simulation loop whenever the current dt
        changes.

        Any logic that should happen exactly once at the start of each datetime
        group should happen here.
        """
        assert isinstance(dt, datetime), \
            "Attempt to set algorithm's current time with non-datetime"
        assert dt.tzinfo == pytz.utc, \
            "Algorithm expects a utc datetime"

        self.datetime = dt
        self.perf_tracker.set_date(dt)
        self.blotter.set_date(dt)

    @api_method
    def get_datetime(self, tz=None):
        """
        Returns the simulation datetime.
        """
        dt = self.datetime
        assert dt.tzinfo == pytz.utc, "Algorithm should have a utc datetime"

        if tz is not None:
            # Convert to the given timezone passed as a string or tzinfo.
            if isinstance(tz, string_types):
                tz = pytz.timezone(tz)
            dt = dt.astimezone(tz)

        return dt  # datetime.datetime objects are immutable.

    def set_transact(self, transact):
        """
        Set the method that will be called to create a
        transaction from open orders and trade events.
        """
        self.blotter.transact = transact

    def update_dividends(self, dividend_frame):
        """
        Set DataFrame used to process dividends.  DataFrame columns should
        contain at least the entries in zp.DIVIDEND_FIELDS.
        """
        self.perf_tracker.update_dividends(dividend_frame)

    @api_method
    def set_slippage(self, slippage):
        if not isinstance(slippage, SlippageModel):
            raise UnsupportedSlippageModel()
        if self.initialized:
            raise OverrideSlippagePostInit()
        self.slippage = slippage

    @api_method
    def set_commission(self, commission):
        if not isinstance(commission, (PerShare, PerTrade, PerDollar)):
            raise UnsupportedCommissionModel()

        if self.initialized:
            raise OverrideCommissionPostInit()
        self.commission = commission

    def set_sources(self, sources):
        assert isinstance(sources, list)
        self.sources = sources

    # Remain backwards compatibility
    @property
    def data_frequency(self):
        return self.sim_params.data_frequency

    @data_frequency.setter
    def data_frequency(self, value):
        assert value in ('daily', 'minute')
        self.sim_params.data_frequency = value

    @api_method
    def order_percent(self, sid, percent,
                      limit_price=None, stop_price=None, style=None):
        """
        Place an order in the specified asset corresponding to the given
        percent of the current portfolio value.

        Note that percent must expressed as a decimal (0.50 means 50\%).
        """
        value = self.portfolio.portfolio_value * percent
        return self.order_value(sid, value,
                                limit_price=limit_price,
                                stop_price=stop_price,
                                style=style)

    @api_method
    def order_target(self, sid, target,
                     limit_price=None, stop_price=None, style=None):
        """
        Place an order to adjust a position to a target number of shares. If
        the position doesn't already exist, this is equivalent to placing a new
        order. If the position does exist, this is equivalent to placing an
        order for the difference between the target number of shares and the
        current number of shares.
        """
        if sid in self.portfolio.positions:
            current_position = self.portfolio.positions[sid].amount
            req_shares = target - current_position
            return self.order(sid, req_shares,
                              limit_price=limit_price,
                              stop_price=stop_price,
                              style=style)
        else:
            return self.order(sid, target,
                              limit_price=limit_price,
                              stop_price=stop_price,
                              style=style)

    @api_method
    def order_target_value(self, sid, target,
                           limit_price=None, stop_price=None, style=None):
        """
        Place an order to adjust a position to a target value. If
        the position doesn't already exist, this is equivalent to placing a new
        order. If the position does exist, this is equivalent to placing an
        order for the difference between the target value and the
        current value.
        If the Asset being ordered is a Future, the 'target value' calculated
        is actually the target exposure, as Futures have no 'value'.
        """
        target_amount = self._calculate_order_value_amount(sid, target)
        return self.order_target(sid, target_amount,
                                 limit_price=limit_price,
                                 stop_price=stop_price,
                                 style=style)

    @api_method
    def order_target_percent(self, sid, target,
                             limit_price=None, stop_price=None, style=None):
        """
        Place an order to adjust a position to a target percent of the
        current portfolio value. If the position doesn't already exist, this is
        equivalent to placing a new order. If the position does exist, this is
        equivalent to placing an order for the difference between the target
        percent and the current percent.

        Note that target must expressed as a decimal (0.50 means 50\%).
        """
        target_value = self.portfolio.portfolio_value * target
        return self.order_target_value(sid, target_value,
                                       limit_price=limit_price,
                                       stop_price=stop_price,
                                       style=style)

    @api_method
    def get_open_orders(self, sid=None):
        if sid is None:
            return {
                key: [order.to_api_obj() for order in orders]
                for key, orders in iteritems(self.blotter.open_orders)
                if orders
            }
        if sid in self.blotter.open_orders:
            orders = self.blotter.open_orders[sid]
            return [order.to_api_obj() for order in orders]
        return []

    @api_method
    def get_order(self, order_id):
        if order_id in self.blotter.orders:
            return self.blotter.orders[order_id].to_api_obj()

    @api_method
    def cancel_order(self, order_param):
        order_id = order_param
        if isinstance(order_param, zipline.protocol.Order):
            order_id = order_param.id

        self.blotter.cancel(order_id)

    @api_method
    def add_history(self, bar_count, frequency, field, ffill=True):
        data_frequency = self.sim_params.data_frequency
        history_spec = HistorySpec(bar_count, frequency, field, ffill,
                                   data_frequency=data_frequency)
        self.history_specs[history_spec.key_str] = history_spec
        if self.initialized:
            if self.history_container:
                self.history_container.ensure_spec(
                    history_spec, self.datetime, self._most_recent_data,
                )
            else:
                self.history_container = self.history_container_class(
                    self.history_specs,
                    self.current_universe(),
                    self.sim_params.first_open,
                    self.sim_params.data_frequency,
                )

    def get_history_spec(self, bar_count, frequency, field, ffill):
        spec_key = HistorySpec.spec_key(bar_count, frequency, field, ffill)
        if spec_key not in self.history_specs:
            data_freq = self.sim_params.data_frequency
            spec = HistorySpec(
                bar_count,
                frequency,
                field,
                ffill,
                data_frequency=data_freq,
            )
            self.history_specs[spec_key] = spec
            if not self.history_container:
                self.history_container = self.history_container_class(
                    self.history_specs,
                    self.current_universe(),
                    self.datetime,
                    self.sim_params.data_frequency,
                    bar_data=self._most_recent_data,
                )
            self.history_container.ensure_spec(
                spec, self.datetime, self._most_recent_data,
            )
        return self.history_specs[spec_key]

    @api_method
    def history(self, bar_count, frequency, field, ffill=True):
        history_spec = self.get_history_spec(
            bar_count,
            frequency,
            field,
            ffill,
        )
        return self.history_container.get_history(history_spec, self.datetime)

    ####################
    # Account Controls #
    ####################

    def register_account_control(self, control):
        """
        Register a new AccountControl to be checked on each bar.
        """
        if self.initialized:
            raise RegisterAccountControlPostInit()
        self.account_controls.append(control)

    def validate_account_controls(self):
        for control in self.account_controls:
            control.validate(self.updated_portfolio(),
                             self.updated_account(),
                             self.get_datetime(),
                             self.trading_client.current_data)

    @api_method
    def set_max_leverage(self, max_leverage=None):
        """
        Set a limit on the maximum leverage of the algorithm.
        """
        control = MaxLeverage(max_leverage)
        self.register_account_control(control)

    ####################
    # Trading Controls #
    ####################

    def register_trading_control(self, control):
        """
        Register a new TradingControl to be checked prior to order calls.
        """
        if self.initialized:
            raise RegisterTradingControlPostInit()
        self.trading_controls.append(control)

    @api_method
    def set_max_position_size(self,
                              sid=None,
                              max_shares=None,
                              max_notional=None):
        """
        Set a limit on the number of shares and/or dollar value held for the
        given sid. Limits are treated as absolute values and are enforced at
        the time that the algo attempts to place an order for sid. This means
        that it's possible to end up with more than the max number of shares
        due to splits/dividends, and more than the max notional due to price
        improvement.

        If an algorithm attempts to place an order that would result in
        increasing the absolute value of shares/dollar value exceeding one of
        these limits, raise a TradingControlException.
        """
        control = MaxPositionSize(asset=sid,
                                  max_shares=max_shares,
                                  max_notional=max_notional)
        self.register_trading_control(control)

    @api_method
    def set_max_order_size(self, sid=None, max_shares=None, max_notional=None):
        """
        Set a limit on the number of shares and/or dollar value of any single
        order placed for sid.  Limits are treated as absolute values and are
        enforced at the time that the algo attempts to place an order for sid.

        If an algorithm attempts to place an order that would result in
        exceeding one of these limits, raise a TradingControlException.
        """
        control = MaxOrderSize(asset=sid,
                               max_shares=max_shares,
                               max_notional=max_notional)
        self.register_trading_control(control)

    @api_method
    def set_max_order_count(self, max_count):
        """
        Set a limit on the number of orders that can be placed within the given
        time interval.
        """
        control = MaxOrderCount(max_count)
        self.register_trading_control(control)

    @api_method
    def set_do_not_order_list(self, restricted_list):
        """
        Set a restriction on which sids can be ordered.
        """
        control = RestrictedListOrder(restricted_list)
        self.register_trading_control(control)

    @api_method
    def set_long_only(self):
        """
        Set a rule specifying that this algorithm cannot take short positions.
        """
        self.register_trading_control(LongOnly())

    ###########
    # FFC API #
    ###########
    @api_method
    @require_not_initialized(AddTermPostInit())
    def add_factor(self, factor, name):
        if name in self._factors:
            raise ValueError("Name %r is already a factor!" % name)
        self._factors[name] = factor

    @api_method
    @require_not_initialized(AddTermPostInit())
    def add_filter(self, filter):
        name = "anon_filter_%d" % len(self._filters)
        self._filters[name] = filter

    # Note: add_classifier is not yet implemented since you can't do anything
    # useful with classifiers yet.

    def _all_terms(self):
        # Merge all three dicts.
        return dict(
            chain.from_iterable(
                iteritems(terms)
                for terms in (self._filters, self._factors, self._classifiers)
            )
        )

    def compute_factor_matrix(self, start_date):
        """
        Compute a factor matrix starting at start_date.
        """
        days = self.trading_environment.trading_days
        start_date_loc = days.get_loc(start_date)
        sim_end = self.sim_params.last_close.normalize()
        end_loc = min(start_date_loc + 252, days.get_loc(sim_end))
        end_date = days[end_loc]
        return self.engine.factor_matrix(
            self._all_terms(),
            start_date,
            end_date,
        ), end_date

    def current_universe(self):
        return self._current_universe

    @classmethod
    def all_api_methods(cls):
        """
        Return a list of all the TradingAlgorithm API methods.
        """
        return [
            fn for fn in itervalues(vars(cls))
            if getattr(fn, 'is_api_method', False)
        ]