import os
import json
import pymongo
import pandas as pd

from datetime import datetime, timedelta
from api import Config, PyApi
from api import BaseDataContainer, History, Bar

from errors import (VNPAST_ConfigError, VNPAST_RequestError,
VNPAST_DataConstructorError, VNPAST_DatabaseError)

class DBConfig(Config):
	"""
	Json-like config object; inherits from Config()

	Contains all kinds of settings relating to database settings.

	privates
	--------
	Inherited from api.Config, plus:

	* client: pymongo.MongoClient object, the connection
	  that is to be used for this session.
	* body: dictionary; the main content of config.

			- client: pymongo.MongoClient(), refers to self.client.

			- dbs: dictionary, is a mapping from database alias
				   to another dictionary, which inclues configurations
				   and themselves(i.e. pymongo.database entity)
				   Concretely, dbs has the structure like:
				   {
				   		alias1 : {
				   			'self': client[dbName1],
				   			'index': dbIndex1,
				   			'collNames': collectionNameType1
				   		},
				   		alias2 : {
				   			'self': client[dbName2],
				   			'index': dbIndex2,
				   			'collNames': collectionNameType2
				   		}, ...
				   }
				   where alias#: string;
				   		 dbs.alias#.self: pymongo.database;
				   		 dbs.alias#.index: string;
				   		 dbs.alias#.collNames: string;

			- dbNames: list; a list of database alias.

	"""
	head = 'DB config'

	client = pymongo.MongoClient()
	body = {
		'client': client,
		'dbs': {
			'EQU_M1': {
				'self': client['DATAYES_EQUITY_M1'],
				'index': 'dateTime',
				'collNames': 'secID'
			},
			'EQU_D1': {
				'self': client['DATAYES_EQUITY_D1'],
				'index': 'date',
				'collNames': 'equTicker'
			},
			'FUT_D1': {
				'self': client['DATAYES_FUTURE_D1'],
				'index': 'date',
				'collNames': 'futTicker'
			},
			'OPT_D1': {
				'self': client['DATAYES_OPTION_D1'],
				'index': 'date',
				'collNames': 'optTicker'
			},
			'FUD_D1': {
				'self': client['DATAYES_FUND_D1'],
				'index': 'date',
				'collNames': 'fudTicker'
			},
			'IDX_D1': {
				'self': client['DATAYES_INDEX_D1'],
				'index': 'date',
				'collNames': 'idxTicker'
			}
		},
		'dbNames': ['EQU_M1', 'EQU_D1', 'FUT_D1',
					'OPT_D1', 'FUD_D1', 'IDX_D1']
	}

	def __init__(self, head=None, token=None, body=None):
		"""
		Inherited constructor.

		parameters
		----------
		* head: string; the name of config file. Default is None.
		* token: string; user's token.
		* body: dictionary; the main content of config
		"""
		super(DBConfig, self).__init__(head, token, body)

	def view(self):
		""" Reloaded Prettify printing method. """
		config_view = {
			'dbConfig_head' : self.head,
			'dbConfig_body' : str(self.body),
		}
		print json.dumps(config_view,
						 indent=4,
						 sort_keys=True)

#----------------------------------------------------------------------
# MongoDB Controller class

class MongodController(object):
	"""
	The MongoDB controller interface.

	MongodController is initialized with a DBConfig configuration
	object and a PyApi object, which has already been contructed with
	its own Config json. The default version of constructor actually
	does nothing special about the database. Yet if user executes shell
	script prepare.sh to prepare the connection, MongodController will
	firstly gather symbols that are going to become collection names
	in corresponding databases. This process is done one database by another,
	user can skip useless databases by editing the scripts.
	Then, it ensures the index of each collection due to the 'index' value
	in DBConfig.body.dbs. Concretely, for D1 bars, the index will be 'date',
	and for intraday bars, it will be 'dateTime'; both take the form of
	datetime.datetime timestamp.

	download() and update() methods of controller dynamically construct
	and maintain the databases, requesting data via PyApi. Once the database
	is constructed, MongodController can access required data via its fetch()
	method.

	privates
	--------
	* _config: DBConfig object; a container of all useful settings for the
	  databases.

	* _api: PyApi object; is responsible for making requests.

	* _client: pymongo.MongoClient object; the connection to MongoDB.

	* _dbs: dictionary; a mapping from database names to another dictionary,
	  which includes configurations of the database and the pymongo.database
	  entity. Inherited from _config.body.['dbs']. Note that keys
	  self._dbs are mere strings, only self._dbs[key]['self'] refers to the
	  pymongo.Database object.

	* _dbNames: list; a list of names of databases.

	* _collNames: dictionary; mapping from self._db[key]['collNames'] attribute
	  to the names of collections(i.e. tickers) within.
	  		- example: _collNames['equTicker'] = ['000001', '000002', ...]

	* _connected: boolean; whether the MongoClient was connected to or not.

	* _mapTickersToSecIDs: dictionary; mapping from stock tickers to
	  its security ID.

	example
	-------
	>> myApi = PyApi(Config())
	>> mydbs = DBConfig()
	>> controller = MongodController(mydbs, myApi)
	>> controller._get_coll_names()
	>> controller._ensure_index()
	>> controller.download_equity_D1(20130101, 20150801)
	>> controller.update_equity_D1()

	"""
	_config = DBConfig()
	_api = None

	_client = None
	_dbs = None
	_dbNames = []
	_collNames = dict()
	_connected = False

	_mapTickersToSecIDs = dict()

	def __init__(self, config, api):
		"""
		Constructor.

		parameters
		----------
		* config: DBConfig object; specifies database configs.
		* api: PyApi object.

		"""
		self._api = api # Set Datayes PyApi.
		if config.body:
			try:
				self._config = config.body
				self._client = config.body['client']
				self._dbs = config.body['dbs']
				self._dbNames = config.body['dbNames']
				self._connected = True
			except KeyError:
				msg = '[MONGOD]: Unable to configure database; ' + \
					  'config file is incomplete.'
				raise VNPAST_ConfigError(msg)
			except Exception,e:
				msg = '[MONGOD]: Unable to configure database; ' + str(e)
				raise VNPAST_ConfigError(msg)

		if self._connected:
			#self._get_coll_names()
			#self._ensure_index()
			pass

	def view(self):
		"""
		NOT IMPLEMENTED
		"""
		return

	#----------------------------------------------------------------------
	# Get collection names methods.

	"""
	Decorator;
	Targeting at path dName, if exists, read data from this file;
	if not, execute handle() which returns a json-like data and
	stores the data at dName path.

	parameters
	----------
	* dName: string; the specific path of file that __md looks at.
	"""
	def __md(dName):
		def _md(get):
			def handle(*args, **kwargs):
				try:
					if os.path.isfile(dName):
						# if directory exists, read from it.
						jsonFile = open(dName,'r')
						data = json.loads(jsonFile.read())
						jsonFile.close()
					else:
						# if not, get data via *get method,
						# then write to the file.
						data = get(*args, **kwargs)
						jsonFile = open(dName, 'w+')
						jsonFile.write(json.dumps(data))
						jsonFile.close()
					#print data
					return data
				except Exception,e:
					raise e
			return handle
		return _md

	@__md('names/equTicker.json')
	def _allEquTickers(self):
		"""get all equity tickers, decorated by @__md()."""
		data = self._api.get_equity_D1()
		allEquTickers = list(data.body['ticker'])
		return allEquTickers

	@__md('names/secID.json')
	def _allSecIds(self):
		"""get all security IDs, decorated by @__md()."""
		data = self._api.get_equity_D1()
		allTickers = list(data.body['ticker'])
		exchangeCDs = list(data.body['exchangeCD'])
		allSecIds = [allTickers[k]+'.'+exchangeCDs[k] for k in range(
					len(allTickers))]
		return allSecIds

	@__md('names/futTicker.json')
	def _allFutTickers(self):
		"""get all future tickers, decorated by @__md()."""
		data = self._api.get_future_D1()
		allFutTickers = list(data.body['ticker'])
		return allFutTickers

	@__md('names/optTicker.json')
	def _allOptTickers(self):
		"""get all option tickers, decorated by @__md()."""
		data = self._api.get_option_D1()
		allOptTickers = list(data.body['ticker'])
		return allOptTickers

	@__md('names/fudTicker.json')
	def _allFudTickers(self):
		"""get all fund tickers, decorated by @__md()."""
		data = self._api.get_fund_D1()
		allFudTickers = list(data.body['ticker'])
		return allFudTickers

	@__md('names/idxTicker.json')
	def _allIdxTickers(self):
		"""get all index tickers, decorated by @__md()."""
		data = self._api.get_index_D1()
		allIdxTickers = list(data.body['ticker'])
		return allIdxTickers

	@__md('names/bndTicker.json')
	def _allBndTickers(self):
		"""get all bond tickers, decorated by @__md()."""
		data = self._api.get_bond_D1()
		allBndTickers = list(data.body['ticker'])
		return allBndTickers

	def _get_coll_names(self):
		"""
		get all instruments'names and store them in self._collNames.

		"""
		try:
			if not os.path.exists('names'):
				os.makedirs('names')

			self._collNames['equTicker'] = self._allEquTickers()
			self._collNames['fudTicker'] = self._allFudTickers()
			self._collNames['secID'] = self._allSecIds()
			self._collNames['futTicker'] = self._allFutTickers()
			self._collNames['optTicker'] = self._allOptTickers()
			self._collNames['idxTicker'] = self._allIdxTickers()

			print '[MONGOD]: Collection names gotten.'
			return 1
		except AssertionError:
			warning = '[MONGOD]: Warning, collection names ' + \
					  'is an empty list.'
			print warning
		except Exception, e:
			msg = '[MONGOD]: Unable to set collection names; ' + \
				   str(e)
			raise VNPAST_DatabaseError(msg)

	#----------------------------------------------------------------------
	# Ensure collection index method.

	def _ensure_index(self):
		"""
		Ensure indices for all databases and collections.

		first access self._dbs config to get index column names;
		then get collection names from self._collNames and loop
		over all collections.

		"""
		if self._collNames and self._dbs:
			try:
				for dbName in self._dbs:
					# Iterate over database configurations.

					db = self._dbs[dbName]
					dbSelf = db['self']
					index = db['index']
					collNames = self._collNames[db['collNames']]
					# db['self'] is the pymongo.Database object.

					for name in collNames:
						coll = dbSelf[name]
						coll.ensure_index([(index,
											pymongo.DESCENDING)], unique=True)
				print '[MONGOD]: MongoDB index set.'
				return 1
			except KeyError:
				msg = '[MONGOD]: Unable to set collection indices; ' + \
					  'infomation in Config.body["dbs"] is incomplete.'
				raise VNPAST_DatabaseError(msg)
			except Exception, e:
				msg = '[MONGOD]: Unable to set collection indices; ' + str(e)
				raise VNPAST_DatabaseError(msg)

	#----------------------------------------------------------------------
	# Download method.

	def download_equity_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['EQU_D1']['self']
			self._api.get_equity_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_equity_M1(self, tasks, startYr=2012, endYr=2015):
		"""

		"""

		try:
			# map equity tickers to security IDs.
			if self._mapTickersToSecIDs:
				maps = self._mapTickersToSecIDs
			else:
				assert os.isfile('./names/secID.json')
				jsonFile = open(dName,'r')
				allSecIds = json.loads(jsonFile.read())
				jsonFile.close()
				allTickers = [s.split('.')[0] for s in allSecIds]
				maps = dict(zip(allTickers, allSecIds))
				self._mapTickersToSecIDs = maps
			tasks_ = [maps[task] for task in tasks]

			db = self._dbs['EQU_M1']['self']
			self._api.get_equity_M1_interMonth(db, id=1,
								     startYr = startYr,
								     endYr = endYr,
								     tasks = tasks_)
		except AssertionError:
			msg = '[MONGOD]: Cannot map tickers to secIDs; ' + \
				  'secID.json does not exist.'
			raise VNPAST_DatabaseError(msg)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_bond_D1(self, start, end, sessionNum=30):
		"""

		"""
		pass

	def download_future_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['FUT_D1']['self']
			self._api.get_future_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_option_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['OPT_D1']['self']
			self._api.get_option_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_index_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['IDX_D1']['self']
			self._api.get_index_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_fund_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['FUD_D1']['self']
			self._api.get_fund_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	#----------------------------------------------------------------------
	# Update methods.

	def __update(self, key, target1, target2, sessionNum):
		"""
		Basic update method.
		Looks into the database specified by 'key', find the latest
		record in the collection of it. Then update the collections
		till last trading date.

		parameters
		----------
		* key: string; a database alias (refer to the database config)
		  e.g., 'EQU_D1'.
		* target1: method; pointer to the function with which controller
		  obtain all tickers in the database. Concretely, target1 are
		  self._all#Tickers methods.
		* target2: method; pointer to the api overlord requesting functions
		  i.e. self._api.get_###_mongod methods.
		* sessionNum: integer; the number of threads.

		"""
		try:
			# get databases and tickers
			db = self._dbs[key]['self']
			index = self._dbs[key]['index']
			allTickers = target1()
			coll = db[allTickers[0]]

			# find the latest timestamp in collection.
			latest = coll.find_one(
					 sort=[(index, pymongo.DESCENDING)])[index]
			start = datetime.strftime(
				latest + timedelta(days=1),'%Y%m%d')
			end = datetime.strftime(datetime.now(), '%Y%m%d')

			# then download.
			target2(db, start, end, sessionNum)
			return db

		except Exception, e:
			msg = '[MONGOD]: Unable to update data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def update_equity_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'EQU_D1',
					  	   target1 = self._allEquTickers,
					  	   target2 = self._api.get_equity_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_future_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'FUT_D1',
					  	   target1 = self._allFutTickers,
					  	   target2 = self._api.get_future_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_option_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'OPT_D1',
					  	   target1 = self._allOptTickers,
					  	   target2 = self._api.get_option_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_index_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'IDX_D1',
					  	   target1 = self._allIdxTickers,
					  	   target2 = self._api.get_index_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_fund_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'FUD_D1',
					  	   target1 = self._allFudTickers,
					  	   target2 = self._api.get_fund_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	#----------------------------------------------------------------------#
	# stuff that will be deprecated

	def update_equity_D1_(self, sessionNum=30):
		"""

		"""
		try:
			# set databases and tickers
			db = self._dbs['EQU_D1']['self']
			index = self._dbs['EQU_D1']['index']
			allEquTickers = self._allEquTickers()
			coll = db[allEquTickers[0]]

			# find the latest timestamp in collection.
			latest = coll.find_one(
					 sort=[(index, pymongo.DESCENDING)])[index]
			start = datetime.strftime(latest + timedelta(days=1),'%Y%m%d')
			end = datetime.strftime(datetime.now(), '%Y%m%d')

			# then download.
			self._api.get_equity_D1_mongod(db, start, end, sessionNum)

		except Exception, e:
			msg = '[MONGOD]: Unable to update data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def update_equity_M1(self):
		"""

		"""
		pass

	#----------------------------------------------------------------------
	# Fetch method.

	def fetch(self, dbName, ticker, start, end, output='list'):
		"""

		"""
		# check inputs' validity.
		if output not in ['df', 'list', 'json']:
			raise ValueError('[MONGOD]: Unsupported output type.')
		if dbName not in self._dbNames:
			raise ValueError('[MONGOD]: Unable to locate database name.')

		db = self._dbs[dbName]
		dbSelf = db['self']
		dbIndex = db['index']
		try:
			coll = db[ticker]
			if len(start)==8 and len(end)==8:
				# yyyymmdd, len()=8
				start = datetime.strptime(start, '%Y%m%d')
				end = datetime.strptime(end, '%Y%m%d')
			elif len(start)==14 and len(end)==14:
				# yyyymmdd HH:MM, len()=14
				start = datetime.strptime(start, '%Y%m%d %H:%M')
				end = datetime.strptime(end, '%Y%m%d %H:%M')
			else:
				pass
			docs = []

			# find in MongoDB.
			for doc in coll.find(filter={dbIndex: {'$lte': end,
				'$gte': start}}, projection={'_id': False}):
				docs.append(doc)

			if output == 'list':
				return docs[::-1]

		except Exception, e:
			msg = '[MONGOD]: Error encountered when fetching data' + \
				  'from MongoDB; '+ str(e)
			return -1

if __name__ == '__main__':
	dc = DBConfig()
	api = PyApi(Config())
	mc = MongodController(dc, api)

	mc.update_index_D1()


"""
=====================================
Multi-class AdaBoosted Decision Trees
=====================================

This example reproduces Figure 1 of Zhu et al [1] and shows how boosting can
improve prediction accuracy on a multi-class problem. The classification
dataset is constructed by taking a ten-dimensional standard normal distribution
and defining three classes separated by nested concentric ten-dimensional
spheres such that roughly equal numbers of samples are in each class (quantiles
of the :math:`\chi^2` distribution).

The performance of the SAMME and SAMME.R [1] algorithms are compared. SAMME.R
uses the probability estimates to update the additive model, while SAMME  uses
the classifications only. As the example illustrates, the SAMME.R algorithm
typically converges faster than SAMME, achieving a lower test error with fewer
boosting iterations. The error of each algorithm on the test set after each
boosting iteration is shown on the left, the classification error on the test
set of each tree is shown in the middle, and the boost weight of each tree is
shown on the right. All trees have a weight of one in the SAMME.R algorithm and
therefore are not shown.

.. [1] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009.

"""
print(__doc__)

# Author: Noel Dawe <noel.dawe@gmail.com>
#
# License: BSD 3 clause

from sklearn.externals.six.moves import zip

import matplotlib.pyplot as plt

from sklearn.datasets import make_gaussian_quantiles
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier

X, y = make_gaussian_quantiles(n_samples=13000, n_features=10,
                               n_classes=3, random_state=1)

n_split = 3000

X_train, X_test = X[:n_split], X[n_split:]
y_train, y_test = y[:n_split], y[n_split:]

bdt_real = AdaBoostClassifier(
    DecisionTreeClassifier(max_depth=2),
    n_estimators=600,
    learning_rate=1)

bdt_discrete = AdaBoostClassifier(
    DecisionTreeClassifier(max_depth=2),
    n_estimators=600,
    learning_rate=1.5,
    algorithm="SAMME")

bdt_real.fit(X_train, y_train)
bdt_discrete.fit(X_train, y_train)

real_test_errors = []
discrete_test_errors = []

for real_test_predict, discrete_train_predict in zip(
        bdt_real.staged_predict(X_test), bdt_discrete.staged_predict(X_test)):
    real_test_errors.append(
        1. - accuracy_score(real_test_predict, y_test))
    discrete_test_errors.append(
        1. - accuracy_score(discrete_train_predict, y_test))

n_trees_discrete = len(bdt_discrete)
n_trees_real = len(bdt_real)

# Boosting might terminate early, but the following arrays are always
# n_estimators long. We crop them to the actual number of trees here:
discrete_estimator_errors = bdt_discrete.estimator_errors_[:n_trees_discrete]
real_estimator_errors = bdt_real.estimator_errors_[:n_trees_real]
discrete_estimator_weights = bdt_discrete.estimator_weights_[:n_trees_discrete]

plt.figure(figsize=(15, 5))

plt.subplot(131)
plt.plot(range(1, n_trees_discrete + 1),
         discrete_test_errors, c='black', label='SAMME')
plt.plot(range(1, n_trees_real + 1),
         real_test_errors, c='black',
         linestyle='dashed', label='SAMME.R')
plt.legend()
plt.ylim(0.18, 0.62)
plt.ylabel('Test Error')
plt.xlabel('Number of Trees')

plt.subplot(132)
plt.plot(range(1, n_trees_discrete + 1), discrete_estimator_errors,
         "b", label='SAMME', alpha=.5)
plt.plot(range(1, n_trees_real + 1), real_estimator_errors,
         "r", label='SAMME.R', alpha=.5)
plt.legend()
plt.ylabel('Error')
plt.xlabel('Number of Trees')
plt.ylim((.2,
         max(real_estimator_errors.max(),
             discrete_estimator_errors.max()) * 1.2))
plt.xlim((-20, len(bdt_discrete) + 20))

plt.subplot(133)
plt.plot(range(1, n_trees_discrete + 1), discrete_estimator_weights,
         "b", label='SAMME')
plt.legend()
plt.ylabel('Weight')
plt.xlabel('Number of Trees')
plt.ylim((0, discrete_estimator_weights.max() * 1.2))
plt.xlim((-20, n_trees_discrete + 20))

# prevent overlapping y-axis labels
plt.subplots_adjust(wspace=0.25)
plt.show()


"""Highest hourly values"""
from collections import OrderedDict
import datetime

import pandas as pd
from pandas.io.sql import read_sql
from matplotlib.font_manager import FontProperties
from pyiem.util import get_autoplot_context, get_dbconn
from pyiem.plot.use_agg import plt
from pyiem.exceptions import NoDataFound

PDICT = OrderedDict(
    [
        ("max_dwpf", "Highest Dew Point Temperature"),
        ("min_dwpf", "Lowest Dew Point Temperature"),
        ("max_tmpf", "Highest Air Temperature"),
        ("min_tmpf", "Lowest Air Temperature"),
        ("max_feel", "Highest Feels Like Temperature"),
        ("min_feel", "Lowest Feels Like Temperature"),
        ("max_mslp", "Maximum Sea Level Pressure"),
        ("min_mslp", "Minimum Sea Level Pressure"),
        ("max_alti", "Maximum Pressure Altimeter"),
        ("min_alti", "Minimum Pressure Altimeter"),
    ]
)
UNITS = {
    "max_dwpf": "F",
    "max_tmpf": "F",
    "min_dwpf": "F",
    "min_tmpf": "F",
    "min_feel": "F",
    "max_feel": "F",
    "max_mslp": "mb",
    "min_mslp": "mb",
    "max_alti": "in",
    "min_alti": "in",
}
MDICT = OrderedDict(
    [
        ("all", "No Month Limit"),
        ("spring", "Spring (MAM)"),
        ("fall", "Fall (SON)"),
        ("winter", "Winter (DJF)"),
        ("summer", "Summer (JJA)"),
        ("gs", "1 May to 30 Sep"),
        ("jan", "January"),
        ("feb", "February"),
        ("mar", "March"),
        ("apr", "April"),
        ("may", "May"),
        ("jun", "June"),
        ("jul", "July"),
        ("aug", "August"),
        ("sep", "September"),
        ("oct", "October"),
        ("nov", "November"),
        ("dec", "December"),
    ]
)

def get_description():
    """ Return a dict describing how to call this plotter """
    desc = dict()
    desc["data"] = True
    desc[
        "description"
    ] = """This table presents the extreme hourly value of
    some variable of your choice based on available observations maintained
    by the IEM.  Sadly, this app will likely point out some bad data points
    as such points tend to be obvious at extremes.  If you contact us to
    point out troubles, we'll certainly attempt to fix the archive to
    remove the bad data points.  Observations are arbitrarly bumped 10
    minutes into the future to place the near to top of the hour obs on
    that hour.  For example, a 9:53 AM observation becomes the ob for 10 AM.
    """
    desc["arguments"] = [
        dict(
            type="zstation",
            name="zstation",
            default="AMW",
            network="IA_ASOS",
            label="Select Station:",
        ),
        dict(
            type="select",
            name="month",
            default="all",
            options=MDICT,
            label="Select Month/Season/All",
        ),
        dict(
            type="select",
            name="var",
            options=PDICT,
            default="max_dwpf",
            label="Which Variable to Plot",
        ),
    ]
    return desc

def plotter(fdict):
    """ Go """
    font0 = FontProperties()
    font0.set_family("monospace")
    font0.set_size(16)
    font1 = FontProperties()
    font1.set_size(16)

    pgconn = get_dbconn("asos")
    ctx = get_autoplot_context(fdict, get_description())
    varname = ctx["var"]
    varname2 = varname.split("_")[1]
    if varname2 in ["dwpf", "tmpf", "feel"]:
        varname2 = "i" + varname2
    month = ctx["month"]
    station = ctx["zstation"]

    if month == "all":
        months = range(1, 13)
    elif month == "fall":
        months = [9, 10, 11]
    elif month == "winter":
        months = [12, 1, 2]
    elif month == "spring":
        months = [3, 4, 5]
    elif month == "summer":
        months = [6, 7, 8]
    elif month == "gs":
        months = [5, 6, 7, 8, 9]
    else:
        ts = datetime.datetime.strptime("2000-" + month + "-01", "%Y-%b-%d")
        # make sure it is length two for the trick below in SQL
        months = [ts.month]

    df = read_sql(
        f"""
    WITH obs as (
        SELECT (valid + '10 minutes'::interval) at time zone %s as ts,
        tmpf::int as itmpf, dwpf::int as idwpf,
        feel::int as ifeel, mslp, alti from alldata
        where station = %s and
        extract(month from valid at time zone %s) in %s),
    agg1 as (
        SELECT extract(hour from ts) as hr,
        max(idwpf) as max_dwpf,
        max(itmpf) as max_tmpf,
        min(idwpf) as min_dwpf,
        min(itmpf) as min_tmpf,
        min(ifeel) as min_feel,
        max(ifeel) as max_feel,
        max(alti) as max_alti,
        min(alti) as min_alti,
        max(mslp) as max_mslp,
        min(mslp) as min_mslp
        from obs GROUP by hr)
    SELECT o.ts, a.hr::int as hr,
        a.{varname} from agg1 a JOIN obs o on
        (a.hr = extract(hour from o.ts)
        and a.{varname} = o.{varname2})
        ORDER by a.hr ASC, o.ts DESC
    """,
        pgconn,
        params=(
            ctx["_nt"].sts[station]["tzname"],
            station,
            ctx["_nt"].sts[station]["tzname"],
            tuple(months),
        ),
        index_col=None,
    )
    if df.empty:
        raise NoDataFound("No Data was found.")
    y0 = 0.1
    yheight = 0.8
    dy = yheight / 24.0
    (fig, ax) = plt.subplots(1, 1, figsize=(8, 8))
    ax.set_position([0.12, y0, 0.57, yheight])
    ax.barh(df["hr"], df[varname], align="center")
    ax.set_ylim(-0.5, 23.5)
    ax.set_yticks([0, 4, 8, 12, 16, 20])
    ax.set_yticklabels(["Mid", "4 AM", "8 AM", "Noon", "4 PM", "8 PM"])
    ax.grid(True)
    ax.set_xlim([df[varname].min() - 5, df[varname].max() + 5])
    ax.set_ylabel(
        "Local Time %s" % (ctx["_nt"].sts[station]["tzname"],),
        fontproperties=font1,
    )

    ab = ctx["_nt"].sts[station]["archive_begin"]
    if ab is None:
        raise NoDataFound("Unknown station metadata")
    fig.text(
        0.5,
        0.93,
        ("%s [%s] %s-%s\n" "%s [%s]")
        % (
            ctx["_nt"].sts[station]["name"],
            station,
            ab.year,
            datetime.date.today().year,
            PDICT[varname],
            MDICT[month],
        ),
        ha="center",
        fontproperties=font1,
    )
    ypos = y0 + (dy / 2.0)
    for hr in range(24):
        sdf = df[df["hr"] == hr]
        if sdf.empty:
            continue
        row = sdf.iloc[0]
        fig.text(
            0.7,
            ypos,
            "%3.0f: %s%s"
            % (
                row[varname],
                pd.Timestamp(row["ts"]).strftime("%d %b %Y"),
                ("*" if len(sdf.index) > 1 else ""),
            ),
            fontproperties=font0,
            va="center",
        )
        ypos += dy
    ax.set_xlabel(
        "%s %s, * denotes ties" % (PDICT[varname], UNITS[varname]),
        fontproperties=font1,
    )

    return plt.gcf(), df

if __name__ == "__main__":
    plotter(dict())


import numpy as np
import matplotlib.pyplot as plt

N = 5000 #number of steps to take
xo = 0.2 #initial position in m
vo = 0.0 #initial velocity
tau = 4.0 #total time for the simulation in s .
dt = tau/float(N) # time step
k = 42.0 #spring constant in N/m
m = 0.25 #mass in kg
g = 9.8 #in m/ s ^2
mu = 0.15 #friction coefficient

y = np.zeros([N,2])

#y is the vector of positions and velocities.

y[0,0] = xo #initial position
y[0,1] = vo #initial velocity

#This function defines the derivatives of the system.
def SpringMass(state,time) :

    g0=state[1]
    if g0 > 0 :
        g1=-k/m*state[0]-g*mu
    else:
        g1=-k/m*state[0]+g*mu

    return np.array([g0,g1])

#This is the basic step in the Euler Method for solving ODEs.
def euler (y,time,dt,derivs) :

    k0 = dt*derivs(y,time)
    ynext = y + k0

    return ynext

for j in range (N-1):

    y[j+1] = euler(y[j],0,dt,SpringMass)

#Just to plot
time = np.linspace(0,tau,N)
plt.plot(time, y[:,0],'b',label="position")
plt.xlabel( "time" )
plt.ylabel( "position" )
plt.savefig('spring_mass.png')


'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on Nov 7, 2011

@author: John Cornwell
@contact: JohnWCornwellV@gmail.com
@summary: Contains utility functions to interact with feature functions in features.py
'''

''' Python imports '''
import math
import pickle
import datetime as dt
from dateutil.relativedelta import relativedelta

''' 3rd Party Imports '''
import numpy as np
import matplotlib.pyplot as plt

''' Our Imports '''
import qstklearn.kdtknn as kdt
from qstkutil import DataAccess as da
from qstkutil import qsdateutil as du
from qstkutil import tsutil as tsu

from qstkfeat.features import *
from qstkfeat.classes import class_fut_ret

def getMarketRel(dData, sRel='$SPX'):
    '''
    @summary: Calculates market relative data.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param sRel - Stock ticker to make the data relative to, $SPX is default.
    @return: Dictionary of market relative values
    '''

    if sRel not in dData['close'].columns:
        raise KeyError('Market relative stock %s not found in getMR()' % sRel)

    dRet = {}

    ''' Make all data market relative, except for volume '''
    for sKey in dData.keys():

        ''' Don't calculate market relative volume, but still copy it over '''
        if sKey == 'volume':
            dRet['volume'] = dData['volume']
            continue

        dfAbsolute = dData[sKey]
        dfRelative = pand.DataFrame(index=dfAbsolute.index, columns=dfAbsolute.columns, data=np.zeros(dfAbsolute.shape))

        ''' Get returns and strip off the market returns '''
        naRets = dfAbsolute.values.copy()
        tsu.returnize0(naRets)

        naMarkRets = naRets[:, list(dfAbsolute.columns).index(sRel)]

        for i, sStock in enumerate(dfAbsolute.columns):
            ''' Don't change the 'market' stock '''
            if sStock == sRel:
                dfRelative.values[:, i] = dfAbsolute.values[:, i]
                continue

            naMarkRel = (naRets[:, i] - naMarkRets) + 1.0

            ''' Find the first non-nan value and start the price at 100 '''
            for j in range(0, dfAbsolute.values.shape[0]):
                if pand.isnull(dfAbsolute.values[j][i]):
                    dfRelative.values[j][i] = float('nan')
                    continue
                dfRelative.values[j][i] = 100
                break

            ''' Now fill prices out using market relative returns '''
            for j in range(j + 1, dfAbsolute.values.shape[0]):
                dfRelative.values[j][i] = dfRelative.values[j - 1][i] * naMarkRel[j]

        ''' Add dataFrame to dictionary to return, move to next key '''
        dRet[sKey] = dfRelative

    return dRet

def applyFeatures(dData, lfcFeatures, ldArgs, sMarketRel=None, sLog=None):
    '''
    @summary: Calculates the feature values using a list of feature functions and arguments.
    @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
    @param lfcFeatures: List of feature functions, most likely coming from features.py
    @param ldArgs: List of dictionaries containing arguments, passed as **kwargs
                   There is a special argument 'MR', if it exists, the data will be made market relative
    @param sMarketRel: If not none, the data will all be made relative to the symbol provided
    @param sLog: If not None, will be filename to log all of the features to
    @return: list of dataframes containing values
    '''

    ldfRet = []

    ''' Calculate market relative data '''
    if sMarketRel is not None:
        dDataRelative = getMarketRel(dData, sRel=sMarketRel)

    ''' Loop though feature functions, pass each data dictionary and arguments '''
    for i, fcFeature in enumerate(lfcFeatures):

        ''' Check for special arguments '''
        if 'MR' in ldArgs[i]:

            if not ldArgs[i]['MR']:
                print 'Warning, setting MR to false will still be Market Relative',\
                      'simply do not include MR key in args'

            if sMarketRel is None:
                raise AssertionError('Functions require market relative stock but sMarketRel=None')
            del ldArgs[i]['MR']
            ldfRet.append(fcFeature(dDataRelative, **ldArgs[i]))
        else:
            ldfRet.append(fcFeature(dData, **ldArgs[i]))

    if not sLog is None:
        with open(sLog, 'wb') as fFile:
            pickle.dump(ldfRet, fFile, -1)

    return ldfRet

def loadFeatures(sLog):
    '''
    @summary: Loads cached features.
    @param sLog: Filename of features.
    @return: Numpy array containing values
    '''

    ldfRet = []

    if not sLog is None:
        with open(sLog, 'rb') as fFile:
            ldfRet = pickle.load(fFile)

    return ldfRet

def stackSyms(ldfFeatures, dtStart=None, dtEnd=None, lsSym=None, sDelNan='ALL', bShowRemoved=False):
    '''
    @summary: Remove symbols from the dataframes, effectively stacking all stocks on top of each other.
    @param ldfFeatures: List of data frames of features.
    @param dtStart: Start time, if None, uses all
    @param dtEnd: End time, if None uses all
    @param lsSym: List of symbols to use, if None, all are used.
    @param sDelNan: Optional, default is ALL: delete any rows with a NaN in it
                    FEAT: Delete if any of the feature points are NaN, allow NaN classification
                    None: Do not delete any NaN rows
    @return: Numpy array containing all features as columns and all
    '''

    if dtStart is None:
        dtStart = ldfFeatures[0].index[0]
    if dtEnd is None:
        dtEnd = ldfFeatures[0].index[-1]

    naRet = None
    ''' Stack stocks vertically '''
    for sStock in ldfFeatures[0].columns:

        if lsSym is not None and sStock not in lsSym:
            continue

        naStkData = None
        ''' Loop through all features, stacking columns horizontally '''
        for dfFeat in ldfFeatures:

            dfFeat = dfFeat.ix[dtStart:dtEnd]

            if naStkData is None:
                naStkData = np.array(dfFeat[sStock].values.reshape(-1, 1))
            else:
                naStkData = np.hstack((naStkData, dfFeat[sStock].values.reshape(-1, 1)))

        ''' Remove nan rows possibly'''
        if 'ALL' == sDelNan or 'FEAT' == sDelNan:
            llValidRows = []
            for i in range(naStkData.shape[0]):

                if 'ALL' == sDelNan and not math.isnan(np.sum(naStkData[i, :])) or \
                   'FEAT' == sDelNan and not math.isnan(np.sum(naStkData[i, :-1])):
                    llValidRows.append(i)
                elif bShowRemoved:
                    print 'Removed', sStock, naStkData[i, :]

            naStkData = naStkData[llValidRows, :]

        ''' Now stack each block of stock data vertically '''
        if naRet is None:
            naRet = naStkData
        else:
            naRet = np.vstack((naRet, naStkData))

    return naRet

def normFeatures(naFeatures, fMin, fMax, bAbsolute, bIgnoreLast=True):
    '''
    @summary: Normalizes the featurespace.
    @param naFeatures:  Numpy array of features,
    @param fMin: Data frame containing the price information for all of the stocks.
    @param fMax: List of feature functions, most likely coming from features.py
    @param bAbsolute: If true, min value will be scaled to fMin, max to fMax, if false,
                      +-1 standard deviations will be scaled to fit between fMin and fMax, i.e. ~69% of the values
    @param bIgnoreLast: If true, last column is ignored (assumed to be classification)
    @return: list of (weights, shifts) to be used to normalize the query points
    '''

    fNewRange = fMax - fMin

    lUseCols = naFeatures.shape[1]
    if bIgnoreLast:
        lUseCols -= 1

    ltRet = []
    ''' Loop through all features '''
    for i in range(lUseCols):

        ''' If absolutely scaled use exact min and max '''
        if bAbsolute:
            fFeatMin = np.min(naFeatures[:, i])
            fFeatMax = np.max(naFeatures[:, i])
        else:
            ''' Otherwise use mean +-1 std deviations for min/max (~94% of data) '''
            fMean = np.average(naFeatures[:, i])
            fStd = np.std(naFeatures[:, i])
            fFeatMin = fMean - fStd
            fFeatMax = fMean + fStd

        ''' Calculate multiplier and shift variable so that new data fits in specified range '''
        fRange = fFeatMax - fFeatMin
        fMult = fNewRange / fRange
        fShift = fMin - (fFeatMin * fMult)

        ''' scale and shift, save in return array '''
        naFeatures[:, i] *= fMult
        naFeatures[:, i] += fShift
        ltRet.append((fMult, fShift))

    return ltRet

def normQuery(naQueries, ltWeightShift):
    '''
    @summary: Normalizes the queries using the given normalization parameters generated from training data.
    @param naQueries:  Numpy array of queries
    @param ltWeightShift: List of weights and shift amounts to be applied to each query.
    @return: None, modifies naQueries
    '''

    assert naQueries.shape[1] == len(ltWeightShift)

    for i in range(naQueries.shape[1]):

        ''' scale and shift, save in return array '''
        naQueries[:, i] *= ltWeightShift[i][0]
        naQueries[:, i] += ltWeightShift[i][1]

def createKnnLearner(naFeatures, lKnn=30, leafsize=10, method='mean'):
    '''
    @summary: Creates a quick KNN learner
    @param naFeatures:  Numpy array of features,
    @param fMin: Data frame containing the price information for all of the stocks.
    @param fMax: List of feature functions, most likely coming from features.py
    @param bAbsolute: If true, min value will be scaled to fMin, max to fMax, if false,
                      +-1 standard deviations will be scaled to fit between fMin and fMax, i.e. ~69% of the values
    @param bIgnoreLast: If true, last column is ignored (assumed to be classification)
    @return: None, data is modified in place
    '''
    cLearner = kdt.kdtknn(k=lKnn, method=method, leafsize=leafsize)

    cLearner.addEvidence(naFeatures)

    return cLearner

def log500(sLog):
    '''
    @summary: Loads cached features.
    @param sLog: Filename of features.
    @return: Nothing, logs features to desired location
    '''

    lsSym = ['A', 'AA', 'AAPL', 'ABC', 'ABT', 'ACE', 'ACN', 'ADBE', 'ADI', 'ADM', 'ADP', 'ADSK', 'AEE', 'AEP', 'AES', 'AET', 'AFL', 'AGN', 'AIG', 'AIV', 'AIZ', 'AKAM', 'AKS', 'ALL', 'ALTR', 'AMAT', 'AMD', 'AMGN', 'AMP', 'AMT', 'AMZN', 'AN', 'ANF', 'ANR', 'AON', 'APA', 'APC', 'APD', 'APH', 'APOL', 'ARG', 'ATI', 'AVB', 'AVP', 'AVY', 'AXP', 'AZO', 'BA', 'BAC', 'BAX', 'BBBY', 'BBT', 'BBY', 'BCR', 'BDX', 'BEN', 'BF.B', 'BHI', 'BIG', 'BIIB', 'BK', 'BLK', 'BLL', 'BMC', 'BMS', 'BMY', 'BRCM', 'BRK.B', 'BSX', 'BTU', 'BXP', 'C', 'CA', 'CAG', 'CAH', 'CAM', 'CAT', 'CB', 'CBG', 'CBS', 'CCE', 'CCL', 'CEG', 'CELG', 'CERN', 'CF', 'CFN', 'CHK', 'CHRW', 'CI', 'CINF', 'CL', 'CLF', 'CLX', 'CMA', 'CMCSA', 'CME', 'CMG', 'CMI', 'CMS', 'CNP', 'CNX', 'COF', 'COG', 'COH', 'COL', 'COP', 'COST', 'COV', 'CPB', 'CPWR', 'CRM', 'CSC', 'CSCO', 'CSX', 'CTAS', 'CTL', 'CTSH', 'CTXS', 'CVC', 'CVH', 'CVS', 'CVX', 'D', 'DD', 'DE', 'DELL', 'DF', 'DFS', 'DGX', 'DHI', 'DHR', 'DIS', 'DISCA', 'DNB', 'DNR', 'DO', 'DOV', 'DOW', 'DPS', 'DRI', 'DTE', 'DTV', 'DUK', 'DV', 'DVA', 'DVN', 'EBAY', 'ECL', 'ED', 'EFX', 'EIX', 'EL', 'EMC', 'EMN', 'EMR', 'EOG', 'EP', 'EQR', 'EQT', 'ERTS', 'ESRX', 'ETFC', 'ETN', 'ETR', 'EW', 'EXC', 'EXPD', 'EXPE', 'F', 'FAST', 'FCX', 'FDO', 'FDX', 'FE', 'FFIV', 'FHN', 'FII', 'FIS', 'FISV', 'FITB', 'FLIR', 'FLR', 'FLS', 'FMC', 'FO', 'FRX', 'FSLR', 'FTI', 'FTR', 'GAS', 'GCI', 'GD', 'GE', 'GILD', 'GIS', 'GLW', 'GME', 'GNW', 'GOOG', 'GPC', 'GPS', 'GR', 'GS', 'GT', 'GWW', 'HAL', 'HAR', 'HAS', 'HBAN', 'HCBK', 'HCN', 'HCP', 'HD', 'HES', 'HIG', 'HNZ', 'HOG', 'HON', 'HOT', 'HP', 'HPQ', 'HRB', 'HRL', 'HRS', 'HSP', 'HST', 'HSY', 'HUM', 'IBM', 'ICE', 'IFF', 'IGT', 'INTC', 'INTU', 'IP', 'IPG', 'IR', 'IRM', 'ISRG', 'ITT', 'ITW', 'IVZ', 'JBL', 'JCI', 'JCP', 'JDSU', 'JEC', 'JNJ', 'JNPR', 'JNS', 'JOYG', 'JPM', 'JWN', 'K', 'KEY', 'KFT', 'KIM', 'KLAC', 'KMB', 'KMX', 'KO', 'KR', 'KSS', 'L', 'LEG', 'LEN', 'LH', 'LIFE', 'LLL', 'LLTC', 'LLY', 'LM', 'LMT', 'LNC', 'LO', 'LOW', 'LSI', 'LTD', 'LUK', 'LUV', 'LXK', 'M', 'MA', 'MAR', 'MAS', 'MAT', 'MCD', 'MCHP', 'MCK', 'MCO', 'MDT', 'MET', 'MHP', 'MHS', 'MJN', 'MKC', 'MMC', 'MMI', 'MMM', 'MO', 'MOLX', 'MON', 'MOS', 'MPC', 'MRK', 'MRO', 'MS', 'MSFT', 'MSI', 'MTB', 'MU', 'MUR', 'MWV', 'MWW', 'MYL', 'NBL', 'NBR', 'NDAQ', 'NE', 'NEE', 'NEM', 'NFLX', 'NFX', 'NI', 'NKE', 'NOC', 'NOV', 'NRG', 'NSC', 'NTAP', 'NTRS', 'NU', 'NUE', 'NVDA', 'NVLS', 'NWL', 'NWSA', 'NYX', 'OI', 'OKE', 'OMC', 'ORCL', 'ORLY', 'OXY', 'PAYX', 'PBCT', 'PBI', 'PCAR', 'PCG', 'PCL', 'PCLN', 'PCP', 'PCS', 'PDCO', 'PEG', 'PEP', 'PFE', 'PFG', 'PG', 'PGN', 'PGR', 'PH', 'PHM', 'PKI', 'PLD', 'PLL', 'PM', 'PNC', 'PNW', 'POM', 'PPG', 'PPL', 'PRU', 'PSA', 'PWR', 'PX', 'PXD', 'QCOM', 'QEP', 'R', 'RAI', 'RDC', 'RF', 'RHI', 'RHT', 'RL', 'ROK', 'ROP', 'ROST', 'RRC', 'RRD', 'RSG', 'RTN', 'S', 'SAI', 'SBUX', 'SCG', 'SCHW', 'SE', 'SEE', 'SHLD', 'SHW', 'SIAL', 'SJM', 'SLB', 'SLE', 'SLM', 'SNA', 'SNDK', 'SNI', 'SO', 'SPG', 'SPLS', 'SRCL', 'SRE', 'STI', 'STJ', 'STT', 'STZ', 'SUN', 'SVU', 'SWK', 'SWN', 'SWY', 'SYK', 'SYMC', 'SYY', 'T', 'TAP', 'TDC', 'TE', 'TEG', 'TEL', 'TER', 'TGT', 'THC', 'TIE', 'TIF', 'TJX', 'TLAB', 'TMK', 'TMO', 'TROW', 'TRV', 'TSN', 'TSO', 'TSS', 'TWC', 'TWX', 'TXN', 'TXT', 'TYC', 'UNH', 'UNM', 'UNP', 'UPS', 'URBN', 'USB', 'UTX', 'V', 'VAR', 'VFC', 'VIA.B', 'VLO', 'VMC', 'VNO', 'VRSN', 'VTR', 'VZ', 'WAG', 'WAT', 'WDC', 'WEC', 'WFC', 'WFM', 'WFR', 'WHR', 'WIN', 'WLP', 'WM', 'WMB', 'WMT', 'WPI', 'WPO', 'WU', 'WY', 'WYN', 'WYNN', 'X', 'XEL', 'XL', 'XLNX', 'XOM', 'XRAY', 'XRX', 'YHOO', 'YUM', 'ZION', 'ZMH']
    lsSym.append('$SPX')
    lsSym.sort()

    ''' Max lookback is 6 months '''
    dtEnd = dt.datetime.now()
    dtEnd = dtEnd.replace(hour=16, minute=0, second=0, microsecond=0)
    dtStart = dtEnd - relativedelta(months=6)

    ''' Pull in current data '''
    norObj = da.DataAccess('Norgate')
    ''' Get 2 extra months for moving averages and future returns '''
    ldtTimestamps = du.getNYSEdays(dtStart - relativedelta(months=2),
                                   dtEnd + relativedelta(months=2), dt.timedelta(hours=16))

    dfPrice = norObj.get_data(ldtTimestamps, lsSym, 'close')
    dfVolume = norObj.get_data(ldtTimestamps, lsSym, 'volume')

    ''' Imported functions from qstkfeat.features, NOTE: last function is classification '''
    lfcFeatures, ldArgs, lsNames = getFeatureFuncs()

    ''' Generate a list of DataFrames, one for each feature, with the same index/column structure as price data '''
    applyFeatures(dfPrice, dfVolume, lfcFeatures, ldArgs, sLog=sLog)

def getFeatureFuncs():
    '''
    @summary: Gets feature functions supported by the website.
    @return: Tuple containing (list of functions, list of arguments, list of names)
    '''

    lfcFeatures = [featMA, featMA, featRSI, featDrawDown, featRunUp, featVolumeDelta, featAroon, featAroon, featStochastic, featBeta, featBollinger, featCorrelation, featPrice, class_fut_ret]
    lsNames = ['MovingAverage', 'RelativeMovingAverage', 'RSI', 'DrawDown', 'RunUp', 'VolumeDelta', 'AroonUp', 'AroonLow', 'Stochastic', 'Beta', 'Bollinger', 'Correlation', 'Price', 'FutureReturn']

    ''' Custom Arguments '''
    ldArgs = [
        {'lLookback':30, 'bRel':False},
        {'lLookback':30, 'bRel':True},
        {'lLookback':14},
        {'lLookback':30},
        {'lLookback':30},
        {'lLookback':30},
        {'bDown':False, 'lLookback':25},
        {'bDown':True, 'lLookback':25},
        {'lLookback':14},
        {'lLookback':14, 'sMarket':'SPY'},
        {'lLookback':20},
        {'lLookback':20, 'sRel':'SPY'},
        {},
        {'lLookforward':5, 'sRel':None, 'bUseOpen':False}
    ]

    return lfcFeatures, ldArgs, lsNames

def testFeature(fcFeature, dArgs):
    '''
    @summary: Quick function to run a feature on some data and plot it to see if it works.
    @param fcFeature: Feature function to test
    @param dArgs: Arguments to pass into feature function
    @return: Void
    '''

    ''' Get Train data for 2009-2010 '''
    dtStart = dt.datetime(2009, 1, 1)
    dtEnd = dt.datetime(2009, 5, 1)

    ''' Pull in current training data and test data '''
    norObj = da.DataAccess('Norgate')
    ''' Get 2 extra months for moving averages and future returns '''
    ldtTimestamps = du.getNYSEdays(dtStart, dtEnd, dt.timedelta(hours=16))

    lsSym = ['GOOG']
    lsSym.append('WMT')
    lsSym.append('$SPX')
    lsSym.append('$VIX')
    lsSym.sort()

    lsKeys = ['open', 'high', 'low', 'close', 'volume']
    ldfData = norObj.get_data(ldtTimestamps, lsSym, lsKeys)
    dData = dict(zip(lsKeys, ldfData))
    dfPrice = dData['close']

    #print dfPrice.values
    ''' Generate a list of DataFrames, one for each feature, with the same index/column structure as price data '''
    dtStart = dt.datetime.now()
    ldfFeatures = applyFeatures(dData, [fcFeature], [dArgs], sMarketRel='$SPX')
    print 'Runtime:', dt.datetime.now() - dtStart

    ''' Use last 3 months of index, to avoid lookback nans '''

    dfPrint = ldfFeatures[0]['GOOG']
    print 'GOOG values:', dfPrint.values
    print 'GOOG Sum:', dfPrint.ix[dfPrint.notnull()].sum()

    for sSym in lsSym:
        plt.subplot(211)
        plt.plot(ldfFeatures[0].index[-60:], dfPrice[sSym].values[-60:])
        plt.plot(ldfFeatures[0].index[-60:], dfPrice['$SPX'].values[-60:] * dfPrice[sSym].values[-60] / dfPrice['$SPX'].values[-60])
        plt.legend((sSym, '$SPX'))
        plt.title(sSym)
        plt.subplot(212)
        plt.plot(ldfFeatures[0].index[-60:], ldfFeatures[0][sSym].values[-60:])
        plt.title('%s-%s' % (fcFeature.__name__, str(dArgs)))
        plt.show()

if __name__ == '__main__':
    pass


# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Compute an expanded vocabulary of embeddings using a word2vec model.

This script loads the word embeddings from a trained skip-thoughts model and
from a trained word2vec model (typically with a larger vocabulary). It trains a
linear regression model without regularization to learn a linear mapping from
the word2vec embedding space to the skip-thoughts embedding space. The model is
then applied to all words in the word2vec vocabulary, yielding vectors in the
skip-thoughts word embedding space for the union of the two vocabularies.

The linear regression task is to learn a parameter matrix W to minimize
  || X - Y * W ||^2,
where X is a matrix of skip-thoughts embeddings of shape [num_words, dim1],
Y is a matrix of word2vec embeddings of shape [num_words, dim2], and W is a
matrix of shape [dim2, dim1].

This is based on the "Translation Matrix" method from the paper:

  "Exploiting Similarities among Languages for Machine Translation"
  Tomas Mikolov, Quoc V. Le, Ilya Sutskever
  https://arxiv.org/abs/1309.4168
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import os.path

import gensim.models
import numpy as np
import sklearn.linear_model
import tensorflow as tf

FLAGS = tf.flags.FLAGS

tf.flags.DEFINE_string("skip_thoughts_model", None,
                       "Checkpoint file or directory containing a checkpoint "
                       "file.")

tf.flags.DEFINE_string("skip_thoughts_vocab", None,
                       "Path to vocabulary file containing a list of newline-"
                       "separated words where the word id is the "
                       "corresponding 0-based index in the file.")

tf.flags.DEFINE_string("word2vec_model", None,
                       "File containing a word2vec model in binary format.")

tf.flags.DEFINE_string("output_dir", None, "Output directory.")

tf.logging.set_verbosity(tf.logging.INFO)

def _load_skip_thoughts_embeddings(checkpoint_path):
  """Loads the embedding matrix from a skip-thoughts model checkpoint.

  Args:
    checkpoint_path: Model checkpoint file or directory containing a checkpoint
        file.

  Returns:
    word_embedding: A numpy array of shape [vocab_size, embedding_dim].

  Raises:
    ValueError: If no checkpoint file matches checkpoint_path.
  """
  if tf.gfile.IsDirectory(checkpoint_path):
    checkpoint_file = tf.train.latest_checkpoint(checkpoint_path)
    if not checkpoint_file:
      raise ValueError("No checkpoint file found in %s" % checkpoint_path)
  else:
    checkpoint_file = checkpoint_path

  tf.logging.info("Loading skip-thoughts embedding matrix from %s",
                  checkpoint_file)
  reader = tf.train.NewCheckpointReader(checkpoint_file)
  word_embedding = reader.get_tensor("word_embedding")
  tf.logging.info("Loaded skip-thoughts embedding matrix of shape %s",
                  word_embedding.shape)

  return word_embedding

def _load_vocabulary(filename):
  """Loads a vocabulary file.

  Args:
    filename: Path to text file containing newline-separated words.

  Returns:
    vocab: A dictionary mapping word to word id.
  """
  tf.logging.info("Reading vocabulary from %s", filename)
  vocab = collections.OrderedDict()
  with tf.gfile.GFile(filename, mode="rb") as f:
    for i, line in enumerate(f):
      word = line.decode("utf-8").strip()
      assert word not in vocab, "Attempting to add word twice: %s" % word
      vocab[word] = i
  tf.logging.info("Read vocabulary of size %d", len(vocab))
  return vocab

def _expand_vocabulary(skip_thoughts_emb, skip_thoughts_vocab, word2vec):
  """Runs vocabulary expansion on a skip-thoughts model using a word2vec model.

  Args:
    skip_thoughts_emb: A numpy array of shape [skip_thoughts_vocab_size,
        skip_thoughts_embedding_dim].
    skip_thoughts_vocab: A dictionary of word to id.
    word2vec: An instance of gensim.models.Word2Vec.

  Returns:
    combined_emb: A dictionary mapping words to embedding vectors.
  """
  # Find words shared between the two vocabularies.
  tf.logging.info("Finding shared words")
  shared_words = [w for w in word2vec.vocab if w in skip_thoughts_vocab]

  # Select embedding vectors for shared words.
  tf.logging.info("Selecting embeddings for %d shared words", len(shared_words))
  shared_st_emb = skip_thoughts_emb[[
      skip_thoughts_vocab[w] for w in shared_words
  ]]
  shared_w2v_emb = word2vec[shared_words]

  # Train a linear regression model on the shared embedding vectors.
  tf.logging.info("Training linear regression model")
  model = sklearn.linear_model.LinearRegression()
  model.fit(shared_w2v_emb, shared_st_emb)

  # Create the expanded vocabulary.
  tf.logging.info("Creating embeddings for expanded vocabuary")
  combined_emb = collections.OrderedDict()
  for w in word2vec.vocab:
    # Ignore words with underscores (spaces).
    if "_" not in w:
      w_emb = model.predict(word2vec[w].reshape(1, -1))
      combined_emb[w] = w_emb.reshape(-1)

  for w in skip_thoughts_vocab:
    combined_emb[w] = skip_thoughts_emb[skip_thoughts_vocab[w]]

  tf.logging.info("Created expanded vocabulary of %d words", len(combined_emb))

  return combined_emb

def main(unused_argv):
  if not FLAGS.skip_thoughts_model:
    raise ValueError("--skip_thoughts_model is required.")
  if not FLAGS.skip_thoughts_vocab:
    raise ValueError("--skip_thoughts_vocab is required.")
  if not FLAGS.word2vec_model:
    raise ValueError("--word2vec_model is required.")
  if not FLAGS.output_dir:
    raise ValueError("--output_dir is required.")

  if not tf.gfile.IsDirectory(FLAGS.output_dir):
    tf.gfile.MakeDirs(FLAGS.output_dir)

  # Load the skip-thoughts embeddings and vocabulary.
  skip_thoughts_emb = _load_skip_thoughts_embeddings(FLAGS.skip_thoughts_model)
  skip_thoughts_vocab = _load_vocabulary(FLAGS.skip_thoughts_vocab)

  # Load the Word2Vec model.
  word2vec = gensim.models.KeyedVectors.load_word2vec_format(
      FLAGS.word2vec_model, binary=True)

  # Run vocabulary expansion.
  embedding_map = _expand_vocabulary(skip_thoughts_emb, skip_thoughts_vocab,
                                     word2vec)

  # Save the output.
  vocab = embedding_map.keys()
  vocab_file = os.path.join(FLAGS.output_dir, "vocab.txt")
  with tf.gfile.GFile(vocab_file, "w") as f:
    f.write("\n".join(vocab))
  tf.logging.info("Wrote vocabulary file to %s", vocab_file)

  embeddings = np.array(embedding_map.values())
  embeddings_file = os.path.join(FLAGS.output_dir, "embeddings.npy")
  np.save(embeddings_file, embeddings)
  tf.logging.info("Wrote embeddings file to %s", embeddings_file)

if __name__ == "__main__":
  tf.app.run()


"""Internal utilties; not for external use
"""
import contextlib
import functools
import itertools
import os.path
import re
import warnings
from collections import OrderedDict
from typing import (
    AbstractSet, Any, Callable, Container, Dict, Hashable, Iterable, Iterator,
    Mapping, MutableMapping, MutableSet, Optional, Sequence, Tuple, TypeVar,
    cast)

import numpy as np
import pandas as pd

from .pycompat import dask_array_type

try:  # Fix typed collections in Python 3.5.0~3.5.2
    from .pycompat import Mapping, MutableMapping, MutableSet  # noqa: F811
except ImportError:
    pass

K = TypeVar('K')
V = TypeVar('V')
T = TypeVar('T')

def _check_inplace(inplace: Optional[bool], default: bool = False) -> bool:
    if inplace is None:
        inplace = default
    else:
        warnings.warn('The inplace argument has been deprecated and will be '
                      'removed in a future version of xarray.',
                      FutureWarning, stacklevel=3)

    return inplace

def alias_message(old_name: str, new_name: str) -> str:
    return '%s has been deprecated. Use %s instead.' % (old_name, new_name)

def alias_warning(old_name: str, new_name: str, stacklevel: int = 3) -> None:
    warnings.warn(alias_message(old_name, new_name), FutureWarning,
                  stacklevel=stacklevel)

def alias(obj: Callable[..., T], old_name: str) -> Callable[..., T]:
    assert isinstance(old_name, str)

    @functools.wraps(obj)
    def wrapper(*args, **kwargs):
        alias_warning(old_name, obj.__name__)
        return obj(*args, **kwargs)
    wrapper.__doc__ = alias_message(old_name, obj.__name__)
    return wrapper

def _maybe_cast_to_cftimeindex(index: pd.Index) -> pd.Index:
    from ..coding.cftimeindex import CFTimeIndex

    if len(index) > 0 and index.dtype == 'O':
        try:
            return CFTimeIndex(index)
        except (ImportError, TypeError):
            return index
    else:
        return index

def safe_cast_to_index(array: Any) -> pd.Index:
    """Given an array, safely cast it to a pandas.Index.

    If it is already a pandas.Index, return it unchanged.

    Unlike pandas.Index, if the array has dtype=object or dtype=timedelta64,
    this function will not attempt to do automatic type conversion but will
    always return an index with dtype=object.
    """
    if isinstance(array, pd.Index):
        index = array
    elif hasattr(array, 'to_index'):
        index = array.to_index()
    else:
        kwargs = {}
        if hasattr(array, 'dtype') and array.dtype.kind == 'O':
            kwargs['dtype'] = object
        index = pd.Index(np.asarray(array), **kwargs)
    return _maybe_cast_to_cftimeindex(index)

def multiindex_from_product_levels(levels: Sequence[pd.Index],
                                   names: Optional[Sequence[str]] = None
                                   ) -> pd.MultiIndex:
    """Creating a MultiIndex from a product without refactorizing levels.

    Keeping levels the same gives back the original labels when we unstack.

    Parameters
    ----------
    levels : sequence of pd.Index
        Values for each MultiIndex level.
    names : optional sequence of objects
        Names for each level.

    Returns
    -------
    pandas.MultiIndex
    """
    if any(not isinstance(lev, pd.Index) for lev in levels):
        raise TypeError('levels must be a list of pd.Index objects')

    split_labels, levels = zip(*[lev.factorize() for lev in levels])
    labels_mesh = np.meshgrid(*split_labels, indexing='ij')
    labels = [x.ravel() for x in labels_mesh]
    return pd.MultiIndex(levels, labels, sortorder=0, names=names)

def maybe_wrap_array(original, new_array):
    """Wrap a transformed array with __array_wrap__ is it can be done safely.

    This lets us treat arbitrary functions that take and return ndarray objects
    like ufuncs, as long as they return an array with the same shape.
    """
    # in case func lost array's metadata
    if isinstance(new_array, np.ndarray) and new_array.shape == original.shape:
        return original.__array_wrap__(new_array)
    else:
        return new_array

def equivalent(first: T, second: T) -> bool:
    """Compare two objects for equivalence (identity or equality), using
    array_equiv if either object is an ndarray
    """
    # TODO: refactor to avoid circular import
    from . import duck_array_ops
    if isinstance(first, np.ndarray) or isinstance(second, np.ndarray):
        return duck_array_ops.array_equiv(first, second)
    else:
        return ((first is second) or
                (first == second) or
                (pd.isnull(first) and pd.isnull(second)))

def peek_at(iterable: Iterable[T]) -> Tuple[T, Iterator[T]]:
    """Returns the first value from iterable, as well as a new iterator with
    the same content as the original iterable
    """
    gen = iter(iterable)
    peek = next(gen)
    return peek, itertools.chain([peek], gen)

def update_safety_check(first_dict: MutableMapping[K, V],
                        second_dict: Mapping[K, V],
                        compat: Callable[[V, V], bool] = equivalent) -> None:
    """Check the safety of updating one dictionary with another.

    Raises ValueError if dictionaries have non-compatible values for any key,
    where compatibility is determined by identity (they are the same item) or
    the `compat` function.

    Parameters
    ----------
    first_dict, second_dict : dict-like
        All items in the second dictionary are checked against for conflicts
        against items in the first dictionary.
    compat : function, optional
        Binary operator to determine if two values are compatible. By default,
        checks for equivalence.
    """
    for k, v in second_dict.items():
        if k in first_dict and not compat(v, first_dict[k]):
            raise ValueError('unsafe to merge dictionaries without '
                             'overriding values; conflicting key %r' % k)

def remove_incompatible_items(first_dict: MutableMapping[K, V],
                              second_dict: Mapping[K, V],
                              compat: Callable[[V, V], bool] = equivalent
                              ) -> None:
    """Remove incompatible items from the first dictionary in-place.

    Items are retained if their keys are found in both dictionaries and the
    values are compatible.

    Parameters
    ----------
    first_dict, second_dict : dict-like
        Mappings to merge.
    compat : function, optional
        Binary operator to determine if two values are compatible. By default,
        checks for equivalence.
    """
    for k in list(first_dict):
        if k not in second_dict or not compat(first_dict[k], second_dict[k]):
            del first_dict[k]

def is_dict_like(value: Any) -> bool:
    return hasattr(value, 'keys') and hasattr(value, '__getitem__')

def is_full_slice(value: Any) -> bool:
    return isinstance(value, slice) and value == slice(None)

def either_dict_or_kwargs(pos_kwargs: Optional[Mapping[Hashable, T]],
                          kw_kwargs: Mapping[str, T],
                          func_name: str
                          ) -> Mapping[Hashable, T]:
    if pos_kwargs is not None:
        if not is_dict_like(pos_kwargs):
            raise ValueError('the first argument to .%s must be a dictionary'
                             % func_name)
        if kw_kwargs:
            raise ValueError('cannot specify both keyword and positional '
                             'arguments to .%s' % func_name)
        return pos_kwargs
    else:
        # Need an explicit cast to appease mypy due to invariance; see
        # https://github.com/python/mypy/issues/6228
        return cast(Mapping[Hashable, T], kw_kwargs)

def is_scalar(value: Any) -> bool:
    """Whether to treat a value as a scalar.

    Any non-iterable, string, or 0-D array
    """
    return (
        getattr(value, 'ndim', None) == 0 or
        isinstance(value, (str, bytes)) or not
        isinstance(value, (Iterable, ) + dask_array_type))

def is_valid_numpy_dtype(dtype: Any) -> bool:
    try:
        np.dtype(dtype)
    except (TypeError, ValueError):
        return False
    else:
        return True

def to_0d_object_array(value: Any) -> np.ndarray:
    """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.
    """
    result = np.empty((), dtype=object)
    result[()] = value
    return result

def to_0d_array(value: Any) -> np.ndarray:
    """Given a value, wrap it in a 0-D numpy.ndarray.
    """
    if np.isscalar(value) or (isinstance(value, np.ndarray) and
                              value.ndim == 0):
        return np.array(value)
    else:
        return to_0d_object_array(value)

def dict_equiv(first: Mapping[K, V], second: Mapping[K, V],
               compat: Callable[[V, V], bool] = equivalent) -> bool:
    """Test equivalence of two dict-like objects. If any of the values are
    numpy arrays, compare them correctly.

    Parameters
    ----------
    first, second : dict-like
        Dictionaries to compare for equality
    compat : function, optional
        Binary operator to determine if two values are compatible. By default,
        checks for equivalence.

    Returns
    -------
    equals : bool
        True if the dictionaries are equal
    """
    for k in first:
        if k not in second or not compat(first[k], second[k]):
            return False
    for k in second:
        if k not in first:
            return False
    return True

def ordered_dict_intersection(first_dict: Mapping[K, V],
                              second_dict: Mapping[K, V],
                              compat: Callable[[V, V], bool] = equivalent
                              ) -> MutableMapping[K, V]:
    """Return the intersection of two dictionaries as a new OrderedDict.

    Items are retained if their keys are found in both dictionaries and the
    values are compatible.

    Parameters
    ----------
    first_dict, second_dict : dict-like
        Mappings to merge.
    compat : function, optional
        Binary operator to determine if two values are compatible. By default,
        checks for equivalence.

    Returns
    -------
    intersection : OrderedDict
        Intersection of the contents.
    """
    new_dict = OrderedDict(first_dict)
    remove_incompatible_items(new_dict, second_dict, compat)
    return new_dict

class SingleSlotPickleMixin:
    """Mixin class to add the ability to pickle objects whose state is defined
    by a single __slots__ attribute. Only necessary under Python 2.
    """
    def __getstate__(self):
        return getattr(self, self.__slots__[0])

    def __setstate__(self, state):
        setattr(self, self.__slots__[0], state)

class Frozen(Mapping[K, V], SingleSlotPickleMixin):
    """Wrapper around an object implementing the mapping interface to make it
    immutable. If you really want to modify the mapping, the mutable version is
    saved under the `mapping` attribute.
    """
    __slots__ = ['mapping']

    def __init__(self, mapping: Mapping[K, V]):
        self.mapping = mapping

    def __getitem__(self, key: K) -> V:
        return self.mapping[key]

    def __iter__(self) -> Iterator[K]:
        return iter(self.mapping)

    def __len__(self) -> int:
        return len(self.mapping)

    def __contains__(self, key: object) -> bool:
        return key in self.mapping

    def __repr__(self) -> str:
        return '%s(%r)' % (type(self).__name__, self.mapping)

def FrozenOrderedDict(*args, **kwargs) -> Frozen:
    return Frozen(OrderedDict(*args, **kwargs))

class SortedKeysDict(MutableMapping[K, V], SingleSlotPickleMixin):
    """An wrapper for dictionary-like objects that always iterates over its
    items in sorted order by key but is otherwise equivalent to the underlying
    mapping.
    """
    __slots__ = ['mapping']

    def __init__(self, mapping: Optional[MutableMapping[K, V]] = None):
        self.mapping = {} if mapping is None else mapping

    def __getitem__(self, key: K) -> V:
        return self.mapping[key]

    def __setitem__(self, key: K, value: V) -> None:
        self.mapping[key] = value

    def __delitem__(self, key: K) -> None:
        del self.mapping[key]

    def __iter__(self) -> Iterator[K]:
        return iter(sorted(self.mapping))

    def __len__(self) -> int:
        return len(self.mapping)

    def __contains__(self, key: object) -> bool:
        return key in self.mapping

    def __repr__(self) -> str:
        return '%s(%r)' % (type(self).__name__, self.mapping)

class OrderedSet(MutableSet[T]):
    """A simple ordered set.

    The API matches the builtin set, but it preserves insertion order of
    elements, like an OrderedDict.
    """
    def __init__(self, values: Optional[AbstractSet[T]] = None):
        self._ordered_dict = OrderedDict()  # type: MutableMapping[T, None]
        if values is not None:
            # Disable type checking - both mypy and PyCharm believes that
            # we're altering the type of self in place (see signature of
            # MutableSet.__ior__)
            self |= values  # type: ignore

    # Required methods for MutableSet

    def __contains__(self, value: object) -> bool:
        return value in self._ordered_dict

    def __iter__(self) -> Iterator[T]:
        return iter(self._ordered_dict)

    def __len__(self) -> int:
        return len(self._ordered_dict)

    def add(self, value: T) -> None:
        self._ordered_dict[value] = None

    def discard(self, value: T) -> None:
        del self._ordered_dict[value]

    # Additional methods

    def update(self, values: AbstractSet[T]) -> None:
        # See comment on __init__ re. type checking
        self |= values  # type: ignore

    def __repr__(self) -> str:
        return '%s(%r)' % (type(self).__name__, list(self))

class NdimSizeLenMixin:
    """Mixin class that extends a class that defines a ``shape`` property to
    one that also defines ``ndim``, ``size`` and ``__len__``.
    """
    @property
    def ndim(self: Any) -> int:
        return len(self.shape)

    @property
    def size(self: Any) -> int:
        # cast to int so that shape = () gives size = 1
        return int(np.prod(self.shape))

    def __len__(self: Any) -> int:
        try:
            return self.shape[0]
        except IndexError:
            raise TypeError('len() of unsized object')

class NDArrayMixin(NdimSizeLenMixin):
    """Mixin class for making wrappers of N-dimensional arrays that conform to
    the ndarray interface required for the data argument to Variable objects.

    A subclass should set the `array` property and override one or more of
    `dtype`, `shape` and `__getitem__`.
    """
    @property
    def dtype(self: Any) -> np.dtype:
        return self.array.dtype

    @property
    def shape(self: Any) -> Tuple[int]:
        return self.array.shape

    def __getitem__(self: Any, key):
        return self.array[key]

    def __repr__(self: Any) -> str:
        return '%s(array=%r)' % (type(self).__name__, self.array)

class ReprObject:
    """Object that prints as the given value, for use with sentinel values.
    """
    def __init__(self, value: str):
        self._value = value

    def __repr__(self) -> str:
        return self._value

@contextlib.contextmanager
def close_on_error(f):
    """Context manager to ensure that a file opened by xarray is closed if an
    exception is raised before the user sees the file object.
    """
    try:
        yield
    except Exception:
        f.close()
        raise

def is_remote_uri(path: str) -> bool:
    return bool(re.search(r'^https?\://', path))

def is_grib_path(path: str) -> bool:
    _, ext = os.path.splitext(path)
    return ext in ['.grib', '.grb', '.grib2', '.grb2']

def is_uniform_spaced(arr, **kwargs) -> bool:
    """Return True if values of an array are uniformly spaced and sorted.

    >>> is_uniform_spaced(range(5))
    True
    >>> is_uniform_spaced([-4, 0, 100])
    False

    kwargs are additional arguments to ``np.isclose``
    """
    arr = np.array(arr, dtype=float)
    diffs = np.diff(arr)
    return bool(np.isclose(diffs.min(), diffs.max(), **kwargs))

def hashable(v: Any) -> bool:
    """Determine whether `v` can be hashed.
    """
    try:
        hash(v)
    except TypeError:
        return False
    return True

def not_implemented(*args, **kwargs):
    return NotImplemented

def decode_numpy_dict_values(attrs: Mapping[K, V]) -> Dict[K, V]:
    """Convert attribute values from numpy objects to native Python objects,
    for use in to_dict
    """
    attrs = dict(attrs)
    for k, v in attrs.items():
        if isinstance(v, np.ndarray):
            attrs[k] = v.tolist()
        elif isinstance(v, np.generic):
            attrs[k] = v.item()
    return attrs

def ensure_us_time_resolution(val):
    """Convert val out of numpy time, for use in to_dict.
    Needed because of numpy bug GH#7619"""
    if np.issubdtype(val.dtype, np.datetime64):
        val = val.astype('datetime64[us]')
    elif np.issubdtype(val.dtype, np.timedelta64):
        val = val.astype('timedelta64[us]')
    return val

class HiddenKeyDict(MutableMapping[K, V]):
    """Acts like a normal dictionary, but hides certain keys.
    """
    # ``__init__`` method required to create instance from class.

    def __init__(self, data: MutableMapping[K, V], hidden_keys: Iterable[K]):
        self._data = data
        self._hidden_keys = frozenset(hidden_keys)

    def _raise_if_hidden(self, key: K) -> None:
        if key in self._hidden_keys:
            raise KeyError('Key `%r` is hidden.' % key)

    # The next five methods are requirements of the ABC.
    def __setitem__(self, key: K, value: V) -> None:
        self._raise_if_hidden(key)
        self._data[key] = value

    def __getitem__(self, key: K) -> V:
        self._raise_if_hidden(key)
        return self._data[key]

    def __delitem__(self, key: K) -> None:
        self._raise_if_hidden(key)
        del self._data[key]

    def __iter__(self) -> Iterator[K]:
        for k in self._data:
            if k not in self._hidden_keys:
                yield k

    def __len__(self) -> int:
        num_hidden = len(self._hidden_keys & self._data.keys())
        return len(self._data) - num_hidden

def get_temp_dimname(dims: Container[Hashable], new_dim: Hashable) -> Hashable:
    """ Get an new dimension name based on new_dim, that is not used in dims.
    If the same name exists, we add an underscore(s) in the head.

    Example1:
        dims: ['a', 'b', 'c']
        new_dim: ['_rolling']
        -> ['_rolling']
    Example2:
        dims: ['a', 'b', 'c', '_rolling']
        new_dim: ['_rolling']
        -> ['__rolling']
    """
    while new_dim in dims:
        new_dim = '_' + str(new_dim)
    return new_dim


"""Abstract tensor product."""

from __future__ import print_function, division

from sympy import Expr, Add, Mul, Matrix, Pow, sympify
from sympy.core.compatibility import u, range
from sympy.core.trace import Tr
from sympy.printing.pretty.stringpict import prettyForm

from sympy.physics.quantum.qexpr import QuantumError
from sympy.physics.quantum.dagger import Dagger
from sympy.physics.quantum.commutator import Commutator
from sympy.physics.quantum.anticommutator import AntiCommutator
from sympy.physics.quantum.state import Ket, Bra
from sympy.physics.quantum.matrixutils import (
    numpy_ndarray,
    scipy_sparse_matrix,
    matrix_tensor_product
)

__all__ = [
    'TensorProduct',
    'tensor_product_simp'
]

#-----------------------------------------------------------------------------
# Tensor product
#-----------------------------------------------------------------------------

_combined_printing = False

def combined_tensor_printing(combined):
    """Set flag controlling whether tensor products of states should be
    printed as a combined bra/ket or as an explicit tensor product of different
    bra/kets. This is a global setting for all TensorProduct class instances.

    Parameters
    ----------
    combine : bool
        When true, tensor product states are combined into one ket/bra, and
        when false explicit tensor product notation is used between each
        ket/bra.
    """
    global _combined_printing
    _combined_printing = combined

class TensorProduct(Expr):
    """The tensor product of two or more arguments.

    For matrices, this uses ``matrix_tensor_product`` to compute the Kronecker
    or tensor product matrix. For other objects a symbolic ``TensorProduct``
    instance is returned. The tensor product is a non-commutative
    multiplication that is used primarily with operators and states in quantum
    mechanics.

    Currently, the tensor product distinguishes between commutative and non-
    commutative arguments.  Commutative arguments are assumed to be scalars and
    are pulled out in front of the ``TensorProduct``. Non-commutative arguments
    remain in the resulting ``TensorProduct``.

    Parameters
    ==========

    args : tuple
        A sequence of the objects to take the tensor product of.

    Examples
    ========

    Start with a simple tensor product of sympy matrices::

        >>> from sympy import I, Matrix, symbols
        >>> from sympy.physics.quantum import TensorProduct

        >>> m1 = Matrix([[1,2],[3,4]])
        >>> m2 = Matrix([[1,0],[0,1]])
        >>> TensorProduct(m1, m2)
        Matrix([
        [1, 0, 2, 0],
        [0, 1, 0, 2],
        [3, 0, 4, 0],
        [0, 3, 0, 4]])
        >>> TensorProduct(m2, m1)
        Matrix([
        [1, 2, 0, 0],
        [3, 4, 0, 0],
        [0, 0, 1, 2],
        [0, 0, 3, 4]])

    We can also construct tensor products of non-commutative symbols:

        >>> from sympy import Symbol
        >>> A = Symbol('A',commutative=False)
        >>> B = Symbol('B',commutative=False)
        >>> tp = TensorProduct(A, B)
        >>> tp
        AxB

    We can take the dagger of a tensor product (note the order does NOT reverse
    like the dagger of a normal product):

        >>> from sympy.physics.quantum import Dagger
        >>> Dagger(tp)
        Dagger(A)xDagger(B)

    Expand can be used to distribute a tensor product across addition:

        >>> C = Symbol('C',commutative=False)
        >>> tp = TensorProduct(A+B,C)
        >>> tp
        (A + B)xC
        >>> tp.expand(tensorproduct=True)
        AxC + BxC
    """
    is_commutative = False

    def __new__(cls, *args):
        if isinstance(args[0], (Matrix, numpy_ndarray, scipy_sparse_matrix)):
            return matrix_tensor_product(*args)
        c_part, new_args = cls.flatten(sympify(args))
        c_part = Mul(*c_part)
        if len(new_args) == 0:
            return c_part
        elif len(new_args) == 1:
            return c_part * new_args[0]
        else:
            tp = Expr.__new__(cls, *new_args)
            return c_part * tp

    @classmethod
    def flatten(cls, args):
        # TODO: disallow nested TensorProducts.
        c_part = []
        nc_parts = []
        for arg in args:
            cp, ncp = arg.args_cnc()
            c_part.extend(list(cp))
            nc_parts.append(Mul._from_args(ncp))
        return c_part, nc_parts

    def _eval_adjoint(self):
        return TensorProduct(*[Dagger(i) for i in self.args])

    def _eval_rewrite(self, pattern, rule, **hints):
        sargs = self.args
        terms = [t._eval_rewrite(pattern, rule, **hints) for t in sargs]
        return TensorProduct(*terms).expand(tensorproduct=True)

    def _sympystr(self, printer, *args):
        from sympy.printing.str import sstr
        length = len(self.args)
        s = ''
        for i in range(length):
            if isinstance(self.args[i], (Add, Pow, Mul)):
                s = s + '('
            s = s + sstr(self.args[i])
            if isinstance(self.args[i], (Add, Pow, Mul)):
                s = s + ')'
            if i != length - 1:
                s = s + 'x'
        return s

    def _pretty(self, printer, *args):

        if (_combined_printing and
                (all([isinstance(arg, Ket) for arg in self.args]) or
                 all([isinstance(arg, Bra) for arg in self.args]))):

            length = len(self.args)
            pform = printer._print('', *args)
            for i in range(length):
                next_pform = printer._print('', *args)
                length_i = len(self.args[i].args)
                for j in range(length_i):
                    part_pform = printer._print(self.args[i].args[j], *args)
                    next_pform = prettyForm(*next_pform.right(part_pform))
                    if j != length_i - 1:
                        next_pform = prettyForm(*next_pform.right(', '))

                if len(self.args[i].args) > 1:
                    next_pform = prettyForm(
                        *next_pform.parens(left='{', right='}'))
                pform = prettyForm(*pform.right(next_pform))
                if i != length - 1:
                    pform = prettyForm(*pform.right(',' + ' '))

            pform = prettyForm(*pform.left(self.args[0].lbracket))
            pform = prettyForm(*pform.right(self.args[0].rbracket))
            return pform

        length = len(self.args)
        pform = printer._print('', *args)
        for i in range(length):
            next_pform = printer._print(self.args[i], *args)
            if isinstance(self.args[i], (Add, Mul)):
                next_pform = prettyForm(
                    *next_pform.parens(left='(', right=')')
                )
            pform = prettyForm(*pform.right(next_pform))
            if i != length - 1:
                if printer._use_unicode:
                    pform = prettyForm(*pform.right(u('\N{N-ARY CIRCLED TIMES OPERATOR}') + u(' ')))
                else:
                    pform = prettyForm(*pform.right('x' + ' '))
        return pform

    def _latex(self, printer, *args):

        if (_combined_printing and
                (all([isinstance(arg, Ket) for arg in self.args]) or
                 all([isinstance(arg, Bra) for arg in self.args]))):

            def _label_wrap(label, nlabels):
                return label if nlabels == 1 else r"\left\{%s\right\}" % label

            s = r", ".join([_label_wrap(arg._print_label_latex(printer, *args),
                                        len(arg.args)) for arg in self.args])

            return r"{%s%s%s}" % (self.args[0].lbracket_latex, s,
                                  self.args[0].rbracket_latex)

        length = len(self.args)
        s = ''
        for i in range(length):
            if isinstance(self.args[i], (Add, Mul)):
                s = s + '\\left('
            # The extra {} brackets are needed to get matplotlib's latex
            # rendered to render this properly.
            s = s + '{' + printer._print(self.args[i], *args) + '}'
            if isinstance(self.args[i], (Add, Mul)):
                s = s + '\\right)'
            if i != length - 1:
                s = s + '\\otimes '
        return s

    def doit(self, **hints):
        return TensorProduct(*[item.doit(**hints) for item in self.args])

    def _eval_expand_tensorproduct(self, **hints):
        """Distribute TensorProducts across addition."""
        args = self.args
        add_args = []
        stop = False
        for i in range(len(args)):
            if isinstance(args[i], Add):
                for aa in args[i].args:
                    tp = TensorProduct(*args[:i] + (aa,) + args[i + 1:])
                    if isinstance(tp, TensorProduct):
                        tp = tp._eval_expand_tensorproduct()
                    add_args.append(tp)
                break

        if add_args:
            return Add(*add_args)
        else:
            return self

    def _eval_trace(self, **kwargs):
        indices = kwargs.get('indices', None)
        exp = tensor_product_simp(self)

        if indices is None or len(indices) == 0:
            return Mul(*[Tr(arg).doit() for arg in exp.args])
        else:
            return Mul(*[Tr(value).doit() if idx in indices else value
                         for idx, value in enumerate(exp.args)])

def tensor_product_simp_Mul(e):
    """Simplify a Mul with TensorProducts.

    Current the main use of this is to simplify a ``Mul`` of ``TensorProduct``s
    to a ``TensorProduct`` of ``Muls``. It currently only works for relatively
    simple cases where the initial ``Mul`` only has scalars and raw
    ``TensorProduct``s, not ``Add``, ``Pow``, ``Commutator``s of
    ``TensorProduct``s.

    Parameters
    ==========

    e : Expr
        A ``Mul`` of ``TensorProduct``s to be simplified.

    Returns
    =======

    e : Expr
        A ``TensorProduct`` of ``Mul``s.

    Examples
    ========

    This is an example of the type of simplification that this function
    performs::

        >>> from sympy.physics.quantum.tensorproduct import \
                    tensor_product_simp_Mul, TensorProduct
        >>> from sympy import Symbol
        >>> A = Symbol('A',commutative=False)
        >>> B = Symbol('B',commutative=False)
        >>> C = Symbol('C',commutative=False)
        >>> D = Symbol('D',commutative=False)
        >>> e = TensorProduct(A,B)*TensorProduct(C,D)
        >>> e
        AxB*CxD
        >>> tensor_product_simp_Mul(e)
        (A*C)x(B*D)

    """
    # TODO: This won't work with Muls that have other composites of
    # TensorProducts, like an Add, Pow, Commutator, etc.
    # TODO: This only works for the equivalent of single Qbit gates.
    if not isinstance(e, Mul):
        return e
    c_part, nc_part = e.args_cnc()
    n_nc = len(nc_part)
    if n_nc == 0 or n_nc == 1:
        return e
    elif e.has(TensorProduct):
        current = nc_part[0]
        if not isinstance(current, TensorProduct):
            raise TypeError('TensorProduct expected, got: %r' % current)
        n_terms = len(current.args)
        new_args = list(current.args)
        for next in nc_part[1:]:
            # TODO: check the hilbert spaces of next and current here.
            if isinstance(next, TensorProduct):
                if n_terms != len(next.args):
                    raise QuantumError(
                        'TensorProducts of different lengths: %r and %r' %
                        (current, next)
                    )
                for i in range(len(new_args)):
                    new_args[i] = new_args[i] * next.args[i]
            else:
                # this won't quite work as we don't want next in the
                # TensorProduct
                for i in range(len(new_args)):
                    new_args[i] = new_args[i] * next
            current = next
        return Mul(*c_part) * TensorProduct(*new_args)
    else:
        return e

def tensor_product_simp(e, **hints):
    """Try to simplify and combine TensorProducts.

    In general this will try to pull expressions inside of ``TensorProducts``.
    It currently only works for relatively simple cases where the products have
    only scalars, raw ``TensorProducts``, not ``Add``, ``Pow``, ``Commutators``
    of ``TensorProducts``. It is best to see what it does by showing examples.

    Examples
    ========

    >>> from sympy.physics.quantum import tensor_product_simp
    >>> from sympy.physics.quantum import TensorProduct
    >>> from sympy import Symbol
    >>> A = Symbol('A',commutative=False)
    >>> B = Symbol('B',commutative=False)
    >>> C = Symbol('C',commutative=False)
    >>> D = Symbol('D',commutative=False)

    First see what happens to products of tensor products:

    >>> e = TensorProduct(A,B)*TensorProduct(C,D)
    >>> e
    AxB*CxD
    >>> tensor_product_simp(e)
    (A*C)x(B*D)

    This is the core logic of this function, and it works inside, powers, sums,
    commutators and anticommutators as well:

    >>> tensor_product_simp(e**2)
    (A*C)x(B*D)**2

    """
    if isinstance(e, Add):
        return Add(*[tensor_product_simp(arg) for arg in e.args])
    elif isinstance(e, Pow):
        return tensor_product_simp(e.base) ** e.exp
    elif isinstance(e, Mul):
        return tensor_product_simp_Mul(e)
    elif isinstance(e, Commutator):
        return Commutator(*[tensor_product_simp(arg) for arg in e.args])
    elif isinstance(e, AntiCommutator):
        return AntiCommutator(*[tensor_product_simp(arg) for arg in e.args])
    else:
        return e


# Even if empty this file is useful so that when running from the root folder
# ./sklearn is added to sys.path by pytest. See
# https://docs.pytest.org/en/latest/pythonpath.html for more details.  For
# example, this allows to build extensions in place and run pytest
# doc/modules/clustering.rst and use sklearn from the local folder rather than
# the one from site-packages.

import platform
from distutils.version import LooseVersion

import pytest
from _pytest.doctest import DoctestItem

from sklearn.utils.fixes import PY3_OR_LATER

PYTEST_MIN_VERSION = '3.3.0'

if LooseVersion(pytest.__version__) < PYTEST_MIN_VERSION:
    raise('Your version of pytest is too old, you should have at least '
          'pytest >= {} installed.'.format(PYTEST_MIN_VERSION))

def pytest_addoption(parser):
    parser.addoption("--skip-network", action="store_true", default=False,
                     help="skip network tests")

def pytest_collection_modifyitems(config, items):

    # FeatureHasher is not compatible with PyPy
    if platform.python_implementation() == 'PyPy':
        skip_marker = pytest.mark.skip(
            reason='FeatureHasher is not compatible with PyPy')
        for item in items:
            if item.name == 'sklearn.feature_extraction.hashing.FeatureHasher':
                item.add_marker(skip_marker)

    # Skip tests which require internet if the flag is provided
    if config.getoption("--skip-network"):
        skip_network = pytest.mark.skip(
            reason="test requires internet connectivity")
        for item in items:
            if "network" in item.keywords:
                item.add_marker(skip_network)

    # numpy changed the str/repr formatting of numpy arrays in 1.14. We want to
    # run doctests only for numpy >= 1.14. We want to skip the doctest for
    # python 2 due to unicode.
    skip_doctests = False
    if not PY3_OR_LATER:
        skip_doctests = True
    try:
        import numpy as np
        if LooseVersion(np.__version__) < LooseVersion('1.14'):
            skip_doctests = True
    except ImportError:
        pass

    if skip_doctests:
        skip_marker = pytest.mark.skip(
            reason='doctests are only run for numpy >= 1.14 and python >= 3')

        for item in items:
            if isinstance(item, DoctestItem):
                item.add_marker(skip_marker)