import os
import json
import pymongo
import pandas as pd

from datetime import datetime, timedelta
from api import Config, PyApi
from api import BaseDataContainer, History, Bar

from errors import (VNPAST_ConfigError, VNPAST_RequestError,
VNPAST_DataConstructorError, VNPAST_DatabaseError)

class DBConfig(Config):
	"""
	Json-like config object; inherits from Config()

	Contains all kinds of settings relating to database settings.

	privates
	--------
	Inherited from api.Config, plus:

	* client: pymongo.MongoClient object, the connection
	  that is to be used for this session.
	* body: dictionary; the main content of config.

			- client: pymongo.MongoClient(), refers to self.client.

			- dbs: dictionary, is a mapping from database alias
				   to another dictionary, which inclues configurations
				   and themselves(i.e. pymongo.database entity)
				   Concretely, dbs has the structure like:
				   {
				   		alias1 : {
				   			'self': client[dbName1],
				   			'index': dbIndex1,
				   			'collNames': collectionNameType1
				   		},
				   		alias2 : {
				   			'self': client[dbName2],
				   			'index': dbIndex2,
				   			'collNames': collectionNameType2
				   		}, ...
				   }
				   where alias#: string;
				   		 dbs.alias#.self: pymongo.database;
				   		 dbs.alias#.index: string;
				   		 dbs.alias#.collNames: string;

			- dbNames: list; a list of database alias.

	"""
	head = 'DB config'

	client = pymongo.MongoClient()
	body = {
		'client': client,
		'dbs': {
			'EQU_M1': {
				'self': client['DATAYES_EQUITY_M1'],
				'index': 'dateTime',
				'collNames': 'secID'
			},
			'EQU_D1': {
				'self': client['DATAYES_EQUITY_D1'],
				'index': 'date',
				'collNames': 'equTicker'
			},
			'FUT_D1': {
				'self': client['DATAYES_FUTURE_D1'],
				'index': 'date',
				'collNames': 'futTicker'
			},
			'OPT_D1': {
				'self': client['DATAYES_OPTION_D1'],
				'index': 'date',
				'collNames': 'optTicker'
			},
			'FUD_D1': {
				'self': client['DATAYES_FUND_D1'],
				'index': 'date',
				'collNames': 'fudTicker'
			},
			'IDX_D1': {
				'self': client['DATAYES_INDEX_D1'],
				'index': 'date',
				'collNames': 'idxTicker'
			}
		},
		'dbNames': ['EQU_M1', 'EQU_D1', 'FUT_D1',
					'OPT_D1', 'FUD_D1', 'IDX_D1']
	}

	def __init__(self, head=None, token=None, body=None):
		"""
		Inherited constructor.

		parameters
		----------
		* head: string; the name of config file. Default is None.
		* token: string; user's token.
		* body: dictionary; the main content of config
		"""
		super(DBConfig, self).__init__(head, token, body)

	def view(self):
		""" Reloaded Prettify printing method. """
		config_view = {
			'dbConfig_head' : self.head,
			'dbConfig_body' : str(self.body),
		}
		print json.dumps(config_view,
						 indent=4,
						 sort_keys=True)

#----------------------------------------------------------------------
# MongoDB Controller class

class MongodController(object):
	"""
	The MongoDB controller interface.

	MongodController is initialized with a DBConfig configuration
	object and a PyApi object, which has already been contructed with
	its own Config json. The default version of constructor actually
	does nothing special about the database. Yet if user executes shell
	script prepare.sh to prepare the connection, MongodController will
	firstly gather symbols that are going to become collection names
	in corresponding databases. This process is done one database by another,
	user can skip useless databases by editing the scripts.
	Then, it ensures the index of each collection due to the 'index' value
	in DBConfig.body.dbs. Concretely, for D1 bars, the index will be 'date',
	and for intraday bars, it will be 'dateTime'; both take the form of
	datetime.datetime timestamp.

	download() and update() methods of controller dynamically construct
	and maintain the databases, requesting data via PyApi. Once the database
	is constructed, MongodController can access required data via its fetch()
	method.

	privates
	--------
	* _config: DBConfig object; a container of all useful settings for the
	  databases.

	* _api: PyApi object; is responsible for making requests.

	* _client: pymongo.MongoClient object; the connection to MongoDB.

	* _dbs: dictionary; a mapping from database names to another dictionary,
	  which includes configurations of the database and the pymongo.database
	  entity. Inherited from _config.body.['dbs']. Note that keys
	  self._dbs are mere strings, only self._dbs[key]['self'] refers to the
	  pymongo.Database object.

	* _dbNames: list; a list of names of databases.

	* _collNames: dictionary; mapping from self._db[key]['collNames'] attribute
	  to the names of collections(i.e. tickers) within.
	  		- example: _collNames['equTicker'] = ['000001', '000002', ...]

	* _connected: boolean; whether the MongoClient was connected to or not.

	* _mapTickersToSecIDs: dictionary; mapping from stock tickers to
	  its security ID.

	example
	-------
	>> myApi = PyApi(Config())
	>> mydbs = DBConfig()
	>> controller = MongodController(mydbs, myApi)
	>> controller._get_coll_names()
	>> controller._ensure_index()
	>> controller.download_equity_D1(20130101, 20150801)
	>> controller.update_equity_D1()

	"""
	_config = DBConfig()
	_api = None

	_client = None
	_dbs = None
	_dbNames = []
	_collNames = dict()
	_connected = False

	_mapTickersToSecIDs = dict()

	def __init__(self, config, api):
		"""
		Constructor.

		parameters
		----------
		* config: DBConfig object; specifies database configs.
		* api: PyApi object.

		"""
		self._api = api # Set Datayes PyApi.
		if config.body:
			try:
				self._config = config.body
				self._client = config.body['client']
				self._dbs = config.body['dbs']
				self._dbNames = config.body['dbNames']
				self._connected = True
			except KeyError:
				msg = '[MONGOD]: Unable to configure database; ' + \
					  'config file is incomplete.'
				raise VNPAST_ConfigError(msg)
			except Exception,e:
				msg = '[MONGOD]: Unable to configure database; ' + str(e)
				raise VNPAST_ConfigError(msg)

		if self._connected:
			#self._get_coll_names()
			#self._ensure_index()
			pass

	def view(self):
		"""
		NOT IMPLEMENTED
		"""
		return

	#----------------------------------------------------------------------
	# Get collection names methods.

	"""
	Decorator;
	Targeting at path dName, if exists, read data from this file;
	if not, execute handle() which returns a json-like data and
	stores the data at dName path.

	parameters
	----------
	* dName: string; the specific path of file that __md looks at.
	"""
	def __md(dName):
		def _md(get):
			def handle(*args, **kwargs):
				try:
					if os.path.isfile(dName):
						# if directory exists, read from it.
						jsonFile = open(dName,'r')
						data = json.loads(jsonFile.read())
						jsonFile.close()
					else:
						# if not, get data via *get method,
						# then write to the file.
						data = get(*args, **kwargs)
						jsonFile = open(dName, 'w+')
						jsonFile.write(json.dumps(data))
						jsonFile.close()
					#print data
					return data
				except Exception,e:
					raise e
			return handle
		return _md

	@__md('names/equTicker.json')
	def _allEquTickers(self):
		"""get all equity tickers, decorated by @__md()."""
		data = self._api.get_equity_D1()
		allEquTickers = list(data.body['ticker'])
		return allEquTickers

	@__md('names/secID.json')
	def _allSecIds(self):
		"""get all security IDs, decorated by @__md()."""
		data = self._api.get_equity_D1()
		allTickers = list(data.body['ticker'])
		exchangeCDs = list(data.body['exchangeCD'])
		allSecIds = [allTickers[k]+'.'+exchangeCDs[k] for k in range(
					len(allTickers))]
		return allSecIds

	@__md('names/futTicker.json')
	def _allFutTickers(self):
		"""get all future tickers, decorated by @__md()."""
		data = self._api.get_future_D1()
		allFutTickers = list(data.body['ticker'])
		return allFutTickers

	@__md('names/optTicker.json')
	def _allOptTickers(self):
		"""get all option tickers, decorated by @__md()."""
		data = self._api.get_option_D1()
		allOptTickers = list(data.body['ticker'])
		return allOptTickers

	@__md('names/fudTicker.json')
	def _allFudTickers(self):
		"""get all fund tickers, decorated by @__md()."""
		data = self._api.get_fund_D1()
		allFudTickers = list(data.body['ticker'])
		return allFudTickers

	@__md('names/idxTicker.json')
	def _allIdxTickers(self):
		"""get all index tickers, decorated by @__md()."""
		data = self._api.get_index_D1()
		allIdxTickers = list(data.body['ticker'])
		return allIdxTickers

	@__md('names/bndTicker.json')
	def _allBndTickers(self):
		"""get all bond tickers, decorated by @__md()."""
		data = self._api.get_bond_D1()
		allBndTickers = list(data.body['ticker'])
		return allBndTickers

	def _get_coll_names(self):
		"""
		get all instruments'names and store them in self._collNames.

		"""
		try:
			if not os.path.exists('names'):
				os.makedirs('names')

			self._collNames['equTicker'] = self._allEquTickers()
			self._collNames['fudTicker'] = self._allFudTickers()
			self._collNames['secID'] = self._allSecIds()
			self._collNames['futTicker'] = self._allFutTickers()
			self._collNames['optTicker'] = self._allOptTickers()
			self._collNames['idxTicker'] = self._allIdxTickers()

			print '[MONGOD]: Collection names gotten.'
			return 1
		except AssertionError:
			warning = '[MONGOD]: Warning, collection names ' + \
					  'is an empty list.'
			print warning
		except Exception, e:
			msg = '[MONGOD]: Unable to set collection names; ' + \
				   str(e)
			raise VNPAST_DatabaseError(msg)

	#----------------------------------------------------------------------
	# Ensure collection index method.

	def _ensure_index(self):
		"""
		Ensure indices for all databases and collections.

		first access self._dbs config to get index column names;
		then get collection names from self._collNames and loop
		over all collections.

		"""
		if self._collNames and self._dbs:
			try:
				for dbName in self._dbs:
					# Iterate over database configurations.

					db = self._dbs[dbName]
					dbSelf = db['self']
					index = db['index']
					collNames = self._collNames[db['collNames']]
					# db['self'] is the pymongo.Database object.

					for name in collNames:
						coll = dbSelf[name]
						coll.ensure_index([(index,
											pymongo.DESCENDING)], unique=True)
				print '[MONGOD]: MongoDB index set.'
				return 1
			except KeyError:
				msg = '[MONGOD]: Unable to set collection indices; ' + \
					  'infomation in Config.body["dbs"] is incomplete.'
				raise VNPAST_DatabaseError(msg)
			except Exception, e:
				msg = '[MONGOD]: Unable to set collection indices; ' + str(e)
				raise VNPAST_DatabaseError(msg)

	#----------------------------------------------------------------------
	# Download method.

	def download_equity_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['EQU_D1']['self']
			self._api.get_equity_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_equity_M1(self, tasks, startYr=2012, endYr=2015):
		"""

		"""

		try:
			# map equity tickers to security IDs.
			if self._mapTickersToSecIDs:
				maps = self._mapTickersToSecIDs
			else:
				assert os.isfile('./names/secID.json')
				jsonFile = open(dName,'r')
				allSecIds = json.loads(jsonFile.read())
				jsonFile.close()
				allTickers = [s.split('.')[0] for s in allSecIds]
				maps = dict(zip(allTickers, allSecIds))
				self._mapTickersToSecIDs = maps
			tasks_ = [maps[task] for task in tasks]

			db = self._dbs['EQU_M1']['self']
			self._api.get_equity_M1_interMonth(db, id=1,
								     startYr = startYr,
								     endYr = endYr,
								     tasks = tasks_)
		except AssertionError:
			msg = '[MONGOD]: Cannot map tickers to secIDs; ' + \
				  'secID.json does not exist.'
			raise VNPAST_DatabaseError(msg)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_bond_D1(self, start, end, sessionNum=30):
		"""

		"""
		pass

	def download_future_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['FUT_D1']['self']
			self._api.get_future_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_option_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['OPT_D1']['self']
			self._api.get_option_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_index_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['IDX_D1']['self']
			self._api.get_index_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def download_fund_D1(self, start, end, sessionNum=30):
		"""

		"""
		try:
			db = self._dbs['FUD_D1']['self']
			self._api.get_fund_D1_mongod(db, start, end, sessionNum)
		except Exception, e:
			msg = '[MONGOD]: Unable to download data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	#----------------------------------------------------------------------
	# Update methods.

	def __update(self, key, target1, target2, sessionNum):
		"""
		Basic update method.
		Looks into the database specified by 'key', find the latest
		record in the collection of it. Then update the collections
		till last trading date.

		parameters
		----------
		* key: string; a database alias (refer to the database config)
		  e.g., 'EQU_D1'.
		* target1: method; pointer to the function with which controller
		  obtain all tickers in the database. Concretely, target1 are
		  self._all#Tickers methods.
		* target2: method; pointer to the api overlord requesting functions
		  i.e. self._api.get_###_mongod methods.
		* sessionNum: integer; the number of threads.

		"""
		try:
			# get databases and tickers
			db = self._dbs[key]['self']
			index = self._dbs[key]['index']
			allTickers = target1()
			coll = db[allTickers[0]]

			# find the latest timestamp in collection.
			latest = coll.find_one(
					 sort=[(index, pymongo.DESCENDING)])[index]
			start = datetime.strftime(
				latest + timedelta(days=1),'%Y%m%d')
			end = datetime.strftime(datetime.now(), '%Y%m%d')

			# then download.
			target2(db, start, end, sessionNum)
			return db

		except Exception, e:
			msg = '[MONGOD]: Unable to update data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def update_equity_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'EQU_D1',
					  	   target1 = self._allEquTickers,
					  	   target2 = self._api.get_equity_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_future_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'FUT_D1',
					  	   target1 = self._allFutTickers,
					  	   target2 = self._api.get_future_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_option_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'OPT_D1',
					  	   target1 = self._allOptTickers,
					  	   target2 = self._api.get_option_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_index_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'IDX_D1',
					  	   target1 = self._allIdxTickers,
					  	   target2 = self._api.get_index_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	def update_fund_D1(self, sessionNum=30):
		"""

		"""
		db = self.__update(key = 'FUD_D1',
					  	   target1 = self._allFudTickers,
					  	   target2 = self._api.get_fund_D1_mongod,
					  	   sessionNum = sessionNum)
		return db

	#----------------------------------------------------------------------#
	# stuff that will be deprecated

	def update_equity_D1_(self, sessionNum=30):
		"""

		"""
		try:
			# set databases and tickers
			db = self._dbs['EQU_D1']['self']
			index = self._dbs['EQU_D1']['index']
			allEquTickers = self._allEquTickers()
			coll = db[allEquTickers[0]]

			# find the latest timestamp in collection.
			latest = coll.find_one(
					 sort=[(index, pymongo.DESCENDING)])[index]
			start = datetime.strftime(latest + timedelta(days=1),'%Y%m%d')
			end = datetime.strftime(datetime.now(), '%Y%m%d')

			# then download.
			self._api.get_equity_D1_mongod(db, start, end, sessionNum)

		except Exception, e:
			msg = '[MONGOD]: Unable to update data; ' + str(e)
			raise VNPAST_DatabaseError(msg)

	def update_equity_M1(self):
		"""

		"""
		pass

	#----------------------------------------------------------------------
	# Fetch method.

	def fetch(self, dbName, ticker, start, end, output='list'):
		"""

		"""
		# check inputs' validity.
		if output not in ['df', 'list', 'json']:
			raise ValueError('[MONGOD]: Unsupported output type.')
		if dbName not in self._dbNames:
			raise ValueError('[MONGOD]: Unable to locate database name.')

		db = self._dbs[dbName]
		dbSelf = db['self']
		dbIndex = db['index']
		try:
			coll = db[ticker]
			if len(start)==8 and len(end)==8:
				# yyyymmdd, len()=8
				start = datetime.strptime(start, '%Y%m%d')
				end = datetime.strptime(end, '%Y%m%d')
			elif len(start)==14 and len(end)==14:
				# yyyymmdd HH:MM, len()=14
				start = datetime.strptime(start, '%Y%m%d %H:%M')
				end = datetime.strptime(end, '%Y%m%d %H:%M')
			else:
				pass
			docs = []

			# find in MongoDB.
			for doc in coll.find(filter={dbIndex: {'$lte': end,
				'$gte': start}}, projection={'_id': False}):
				docs.append(doc)

			if output == 'list':
				return docs[::-1]

		except Exception, e:
			msg = '[MONGOD]: Error encountered when fetching data' + \
				  'from MongoDB; '+ str(e)
			return -1

if __name__ == '__main__':
	dc = DBConfig()
	api = PyApi(Config())
	mc = MongodController(dc, api)

	mc.update_index_D1()


"""
=============================================================================
Manifold learning on handwritten digits: Locally Linear Embedding, Isomap...
=============================================================================

An illustration of various embeddings on the digits dataset.

The RandomTreesEmbedding, from the :mod:`sklearn.ensemble` module, is not
technically a manifold embedding method, as it learn a high-dimensional
representation on which we apply a dimensionality reduction method.
However, it is often useful to cast a dataset into a representation in
which the classes are linearly-separable.

t-SNE will be initialized with the embedding that is generated by PCA in
this example, which is not the default setting. It ensures global stability
of the embedding, i.e., the embedding does not depend on random
initialization.
"""

# Authors: Fabian Pedregosa <fabian.pedregosa@inria.fr>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Gael Varoquaux
# License: BSD 3 clause (C) INRIA 2011

print(__doc__)
from time import time

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import offsetbox
from sklearn import (manifold, datasets, decomposition, ensemble,
                     discriminant_analysis, random_projection)

digits = datasets.load_digits(n_class=6)
X = digits.data
y = digits.target
n_samples, n_features = X.shape
n_neighbors = 30

#----------------------------------------------------------------------
# Scale and visualize the embedding vectors
def plot_embedding(X, title=None):
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)

    plt.figure()
    ax = plt.subplot(111)
    for i in range(X.shape[0]):
        plt.text(X[i, 0], X[i, 1], str(digits.target[i]),
                 color=plt.cm.Set1(y[i] / 10.),
                 fontdict={'weight': 'bold', 'size': 9})

    if hasattr(offsetbox, 'AnnotationBbox'):
        # only print thumbnails with matplotlib > 1.0
        shown_images = np.array([[1., 1.]])  # just something big
        for i in range(digits.data.shape[0]):
            dist = np.sum((X[i] - shown_images) ** 2, 1)
            if np.min(dist) < 4e-3:
                # don't show points that are too close
                continue
            shown_images = np.r_[shown_images, [X[i]]]
            imagebox = offsetbox.AnnotationBbox(
                offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r),
                X[i])
            ax.add_artist(imagebox)
    plt.xticks([]), plt.yticks([])
    if title is not None:
        plt.title(title)

#----------------------------------------------------------------------
# Plot images of the digits
n_img_per_row = 20
img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row))
for i in range(n_img_per_row):
    ix = 10 * i + 1
    for j in range(n_img_per_row):
        iy = 10 * j + 1
        img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8))

plt.imshow(img, cmap=plt.cm.binary)
plt.xticks([])
plt.yticks([])
plt.title('A selection from the 64-dimensional digits dataset')

#----------------------------------------------------------------------
# Random 2D projection using a random unitary matrix
print("Computing random projection")
rp = random_projection.SparseRandomProjection(n_components=2, random_state=42)
X_projected = rp.fit_transform(X)
plot_embedding(X_projected, "Random Projection of the digits")

#----------------------------------------------------------------------
# Projection on to the first 2 principal components

print("Computing PCA projection")
t0 = time()
X_pca = decomposition.TruncatedSVD(n_components=2).fit_transform(X)
plot_embedding(X_pca,
               "Principal Components projection of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Projection on to the first 2 linear discriminant components

print("Computing Linear Discriminant Analysis projection")
X2 = X.copy()
X2.flat[::X.shape[1] + 1] += 0.01  # Make X invertible
t0 = time()
X_lda = discriminant_analysis.LinearDiscriminantAnalysis(n_components=2).fit_transform(X2, y)
plot_embedding(X_lda,
               "Linear Discriminant projection of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Isomap projection of the digits dataset
print("Computing Isomap embedding")
t0 = time()
X_iso = manifold.Isomap(n_neighbors, n_components=2).fit_transform(X)
print("Done.")
plot_embedding(X_iso,
               "Isomap projection of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Locally linear embedding of the digits dataset
print("Computing LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='standard')
t0 = time()
X_lle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_lle,
               "Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Modified Locally linear embedding of the digits dataset
print("Computing modified LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='modified')
t0 = time()
X_mlle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_mlle,
               "Modified Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# HLLE embedding of the digits dataset
print("Computing Hessian LLE embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='hessian')
t0 = time()
X_hlle = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_hlle,
               "Hessian Locally Linear Embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# LTSA embedding of the digits dataset
print("Computing LTSA embedding")
clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2,
                                      method='ltsa')
t0 = time()
X_ltsa = clf.fit_transform(X)
print("Done. Reconstruction error: %g" % clf.reconstruction_error_)
plot_embedding(X_ltsa,
               "Local Tangent Space Alignment of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# MDS  embedding of the digits dataset
print("Computing MDS embedding")
clf = manifold.MDS(n_components=2, n_init=1, max_iter=100)
t0 = time()
X_mds = clf.fit_transform(X)
print("Done. Stress: %f" % clf.stress_)
plot_embedding(X_mds,
               "MDS embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Random Trees embedding of the digits dataset
print("Computing Totally Random Trees embedding")
hasher = ensemble.RandomTreesEmbedding(n_estimators=200, random_state=0,
                                       max_depth=5)
t0 = time()
X_transformed = hasher.fit_transform(X)
pca = decomposition.TruncatedSVD(n_components=2)
X_reduced = pca.fit_transform(X_transformed)

plot_embedding(X_reduced,
               "Random forest embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# Spectral embedding of the digits dataset
print("Computing Spectral embedding")
embedder = manifold.SpectralEmbedding(n_components=2, random_state=0,
                                      eigen_solver="arpack")
t0 = time()
X_se = embedder.fit_transform(X)

plot_embedding(X_se,
               "Spectral embedding of the digits (time %.2fs)" %
               (time() - t0))

#----------------------------------------------------------------------
# t-SNE embedding of the digits dataset
print("Computing t-SNE embedding")
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
t0 = time()
X_tsne = tsne.fit_transform(X)

plot_embedding(X_tsne,
               "t-SNE embedding of the digits (time %.2fs)" %
               (time() - t0))

plt.show()


from __future__ import absolute_import
from __future__ import print_function

import json
import os
import pandas as pd
from PIL import Image
import shutil
from six.moves.urllib.request import urlopen
import subprocess
import tempfile
import unittest
import uuid

import tensorflow as tf
from tensorflow.python.lib.io import file_io

import google.datalab as dl
import google.datalab.bigquery as bq
import google.datalab.storage as storage

CODE_PATH = os.path.abspath(os.path.join(
    os.path.dirname(__file__), '..', 'mltoolbox', 'code_free_ml'))

# Some tests put files in GCS or use BigQuery. If HAS_CREDENTIALS is false,
# those tests will not run.
HAS_CREDENTIALS = True
try:
  dl.Context.default().project_id
except Exception:
  HAS_CREDENTIALS = False

class TestTransformRawData(unittest.TestCase):
  """Tests for applying a saved model"""

  @classmethod
  def setUpClass(cls):

    # Set up dirs.
    cls.working_dir = tempfile.mkdtemp()
    cls.source_dir = os.path.join(cls.working_dir, 'source')
    cls.analysis_dir = os.path.join(cls.working_dir, 'analysis')
    cls.output_dir = os.path.join(cls.working_dir, 'output')
    file_io.create_dir(cls.source_dir)

    # Make test image files.
    img1_file = os.path.join(cls.source_dir, 'img1.jpg')
    image1 = Image.new('RGBA', size=(300, 300), color=(155, 0, 0))
    image1.save(img1_file)
    img2_file = os.path.join(cls.source_dir, 'img2.jpg')
    image2 = Image.new('RGBA', size=(50, 50), color=(125, 240, 0))
    image2.save(img2_file)
    img3_file = os.path.join(cls.source_dir, 'img3.jpg')
    image3 = Image.new('RGBA', size=(800, 600), color=(33, 55, 77))
    image3.save(img3_file)

    # Download inception checkpoint. Note that gs url doesn't work because
    # we may not have gcloud signed in when running the test.
    url = ('https://storage.googleapis.com/cloud-ml-data/img/' +
           'flower_photos/inception_v3_2016_08_28.ckpt')
    checkpoint_path = os.path.join(cls.working_dir, "checkpoint")
    response = urlopen(url)
    with open(checkpoint_path, 'wb') as f:
      f.write(response.read())

    # Make csv input file
    cls.csv_input_filepath = os.path.join(cls.source_dir, 'input.csv')
    file_io.write_string_to_file(
        cls.csv_input_filepath,
        '1,1,Monday,23.0,%s\n' % img1_file +
        '2,0,Friday,18.0,%s\n' % img2_file +
        '3,0,Sunday,12.0,%s\n' % img3_file)

    # Call analyze.py to create analysis results.
    schema = [{'name': 'key_col', 'type': 'INTEGER'},
              {'name': 'target_col', 'type': 'FLOAT'},
              {'name': 'cat_col', 'type': 'STRING'},
              {'name': 'num_col', 'type': 'FLOAT'},
              {'name': 'img_col', 'type': 'STRING'}]
    schema_file = os.path.join(cls.source_dir, 'schema.json')
    file_io.write_string_to_file(schema_file, json.dumps(schema))
    features = {'key_col': {'transform': 'key'},
                'target_col': {'transform': 'target'},
                'cat_col': {'transform': 'one_hot'},
                'num_col': {'transform': 'identity'},
                'img_col': {'transform': 'image_to_vec', 'checkpoint': checkpoint_path}}
    features_file = os.path.join(cls.source_dir, 'features.json')
    file_io.write_string_to_file(features_file, json.dumps(features))
    cmd = ['python ' + os.path.join(CODE_PATH, 'analyze.py'),
           '--output=' + cls.analysis_dir,
           '--csv=' + cls.csv_input_filepath,
           '--schema=' + schema_file,
           '--features=' + features_file]
    subprocess.check_call(' '.join(cmd), shell=True)

  @classmethod
  def tearDownClass(cls):
    shutil.rmtree(cls.working_dir)

  def test_local_csv_transform(self):
    """Test transfrom from local csv files."""

    cmd = ['python ' + os.path.join(CODE_PATH, 'transform.py'),
           '--csv=' + self.csv_input_filepath,
           '--analysis=' + self.analysis_dir,
           '--prefix=features',
           '--output=' + self.output_dir]
    print('cmd ', ' '.join(cmd))
    subprocess.check_call(' '.join(cmd), shell=True)

    # Read the tf record file. There should only be one file.
    record_filepath = os.path.join(self.output_dir,
                                   'features-00000-of-00001.tfrecord.gz')
    options = tf.python_io.TFRecordOptions(
        compression_type=tf.python_io.TFRecordCompressionType.GZIP)
    serialized_examples = list(tf.python_io.tf_record_iterator(record_filepath, options=options))
    self.assertEqual(len(serialized_examples), 3)

    # Find the example with key=1 in the file.
    first_example = None
    for ex in serialized_examples:
      example = tf.train.Example()
      example.ParseFromString(ex)
      if example.features.feature['key_col'].int64_list.value[0] == 1:
        first_example = example
    self.assertIsNotNone(first_example)

    transformed_number = first_example.features.feature['num_col'].float_list.value[0]
    self.assertAlmostEqual(transformed_number, 23.0)

    # transformed category = row number in the vocab file.
    transformed_category = first_example.features.feature['cat_col'].int64_list.value[0]
    vocab = pd.read_csv(
        os.path.join(self.analysis_dir, 'vocab_cat_col.csv'),
        header=None,
        names=['label', 'count'],
        dtype=str)
    origional_category = vocab.iloc[transformed_category]['label']
    self.assertEqual(origional_category, 'Monday')

    image_bytes = first_example.features.feature['img_col'].float_list.value
    self.assertEqual(len(image_bytes), 2048)
    self.assertTrue(any(x != 0.0 for x in image_bytes))

  @unittest.skipIf(not HAS_CREDENTIALS, 'GCS access missing')
  def test_local_bigquery_transform(self):
    """Test transfrom locally, but the data comes from bigquery."""

    # Make a BQ table, and insert 1 row.
    try:
      bucket_name = 'temp_pydatalab_test_%s' % uuid.uuid4().hex
      bucket_root = 'gs://%s' % bucket_name
      bucket = storage.Bucket(bucket_name)
      bucket.create()

      project_id = dl.Context.default().project_id

      dataset_name = 'test_transform_raw_data_%s' % uuid.uuid4().hex
      table_name = 'tmp_table'

      dataset = bq.Dataset((project_id, dataset_name)).create()
      table = bq.Table((project_id, dataset_name, table_name))
      table.create([{'name': 'key_col', 'type': 'INTEGER'},
                    {'name': 'target_col', 'type': 'FLOAT'},
                    {'name': 'cat_col', 'type': 'STRING'},
                    {'name': 'num_col', 'type': 'FLOAT'},
                    {'name': 'img_col', 'type': 'STRING'}])

      img1_file = os.path.join(self.source_dir, 'img1.jpg')
      dest_file = os.path.join(bucket_root, 'img1.jpg')
      file_io.copy(img1_file, dest_file)

      data = [
          {
           'key_col': 1,
           'target_col': 1.0,
           'cat_col': 'Monday',
           'num_col': 23.0,
           'img_col': dest_file,
          },
      ]
      table.insert(data=data)

      cmd = ['python ' + os.path.join(CODE_PATH, 'transform.py'),
             '--bigquery=%s.%s.%s' % (project_id, dataset_name, table_name),
             '--analysis=' + self.analysis_dir,
             '--prefix=features',
             '--project-id=' + project_id,
             '--output=' + self.output_dir]
      print('cmd ', ' '.join(cmd))
      subprocess.check_call(' '.join(cmd), shell=True)

      # Read the tf record file. There should only be one file.
      record_filepath = os.path.join(self.output_dir,
                                     'features-00000-of-00001.tfrecord.gz')
      options = tf.python_io.TFRecordOptions(
          compression_type=tf.python_io.TFRecordCompressionType.GZIP)
      serialized_examples = list(tf.python_io.tf_record_iterator(record_filepath, options=options))
      self.assertEqual(len(serialized_examples), 1)

      example = tf.train.Example()
      example.ParseFromString(serialized_examples[0])

      transformed_number = example.features.feature['num_col'].float_list.value[0]
      self.assertAlmostEqual(transformed_number, 23.0)
      transformed_category = example.features.feature['cat_col'].int64_list.value[0]
      self.assertEqual(transformed_category, 2)
      image_bytes = example.features.feature['img_col'].float_list.value
      self.assertEqual(len(image_bytes), 2048)
      self.assertTrue(any(x != 0.0 for x in image_bytes))
    finally:
      dataset.delete(delete_contents=True)

      for obj in bucket.objects():
        obj.delete()
      bucket.delete()

if __name__ == '__main__':
    unittest.main()


07 Selenium, more beautifulsoup/zh_wie_neu.py

# coding: utf-8

from bs4 import BeautifulSoup
import pandas as pd
import requests
import time
import progressbar

bar = progressbar.ProgressBar()
lst = []
lst_pass = []

for elem,i in zip(range(1697,13000), bar((range(1697,13000)))):

    url = "https://www.zueriwieneu.ch/report/" + str(elem)
    response = requests.get(url)
    züri_soup = BeautifulSoup(response.text, 'html.parser')

    if züri_soup.find('h1').text != 'Melden Sie Schäden an der Infrastruktur von Zürich':
        Mini_dict = {
            'Kategorie' : züri_soup.find('h1').text,
            'Meldedatum' : züri_soup.find('div', {'class':'problem-header clearfix'}).find('p').text.strip(),
            'Meldung' : züri_soup.find('div', {'class':'problem-header clearfix'}).find_all('p')[1],
            'Antwortdatum' : züri_soup.find('ul', {'class':'item-list item-list--updates'}).find_all('p')[0].text,
            'Antwort' : züri_soup.find('ul', {'class':'item-list item-list--updates'}).find_all('p')[1].text,
            'URL' : url,
            'Lat' : float(züri_soup.find('div', {'id':'js-map-data'}).get('data-latitude')),
            'Long': float(züri_soup.find('div', {'id':'js-map-data'}).get('data-longitude'))
            }
        lst.append(Mini_dict)
    else:
        lst_pass.append(url)

date = time.strftime("%Y-%m-%d%H:%M:%S")
pd.DataFrame(lst).to_csv(date+'züriwieneu.csv')


"""
=================================================
Generate a functional label from source estimates
=================================================

Threshold source estimates and produce a functional label. The label
is typically the region of interest that contains high values.
Here we compare the average time course in the anatomical label obtained
by FreeSurfer segmentation and the average time course from the
functional label. As expected the time course in the functional
label yields higher values.
"""
# Author: Luke Bloy <luke.bloy@gmail.com>
#         Alex Gramfort <alexandre.gramfort@inria.fr>
# License: BSD (3-clause)

import numpy as np
import matplotlib.pyplot as plt

import mne
from mne.minimum_norm import read_inverse_operator, apply_inverse
from mne.datasets import sample

print(__doc__)

data_path = sample.data_path()
fname_inv = data_path + '/MEG/sample/sample_audvis-meg-oct-6-meg-inv.fif'
fname_evoked = data_path + '/MEG/sample/sample_audvis-ave.fif'
subjects_dir = data_path + '/subjects'
subject = 'sample'

snr = 3.0
lambda2 = 1.0 / snr ** 2
method = "dSPM"  # use dSPM method (could also be MNE or sLORETA)

# Compute a label/ROI based on the peak power between 80 and 120 ms.
# The label bankssts-lh is used for the comparison.
aparc_label_name = 'bankssts-lh'
tmin, tmax = 0.080, 0.120

# Load data
evoked = mne.read_evokeds(fname_evoked, condition=0, baseline=(None, 0))
inverse_operator = read_inverse_operator(fname_inv)
src = inverse_operator['src']  # get the source space

# Compute inverse solution
stc = apply_inverse(evoked, inverse_operator, lambda2, method,
                    pick_ori='normal')

# Make an STC in the time interval of interest and take the mean
stc_mean = stc.copy().crop(tmin, tmax).mean()

# use the stc_mean to generate a functional label
# region growing is halted at 60% of the peak value within the
# anatomical label / ROI specified by aparc_label_name
label = mne.read_labels_from_annot(subject, parc='aparc',
                                   subjects_dir=subjects_dir,
                                   regexp=aparc_label_name)[0]
stc_mean_label = stc_mean.in_label(label)
data = np.abs(stc_mean_label.data)
stc_mean_label.data[data < 0.6 * np.max(data)] = 0.

# 8.5% of original source space vertices were omitted during forward
# calculation, suppress the warning here with verbose='error'
func_labels, _ = mne.stc_to_label(stc_mean_label, src=src, smooth=True,
                                  subjects_dir=subjects_dir, connected=True,
                                  verbose='error')

# take first as func_labels are ordered based on maximum values in stc
func_label = func_labels[0]

# load the anatomical ROI for comparison
anat_label = mne.read_labels_from_annot(subject, parc='aparc',
                                        subjects_dir=subjects_dir,
                                        regexp=aparc_label_name)[0]

# extract the anatomical time course for each label
stc_anat_label = stc.in_label(anat_label)
pca_anat = stc.extract_label_time_course(anat_label, src, mode='pca_flip')[0]

stc_func_label = stc.in_label(func_label)
pca_func = stc.extract_label_time_course(func_label, src, mode='pca_flip')[0]

# flip the pca so that the max power between tmin and tmax is positive
pca_anat *= np.sign(pca_anat[np.argmax(np.abs(pca_anat))])
pca_func *= np.sign(pca_func[np.argmax(np.abs(pca_anat))])

###############################################################################
# plot the time courses....
plt.figure()
plt.plot(1e3 * stc_anat_label.times, pca_anat, 'k',
         label='Anatomical %s' % aparc_label_name)
plt.plot(1e3 * stc_func_label.times, pca_func, 'b',
         label='Functional %s' % aparc_label_name)
plt.legend()
plt.show()

###############################################################################
# plot brain in 3D with PySurfer if available
brain = stc_mean.plot(hemi='lh', subjects_dir=subjects_dir)
brain.show_view('lateral')

# show both labels
brain.add_label(anat_label, borders=True, color='k')
brain.add_label(func_label, borders=True, color='b')


"""Isomap for manifold learning"""

# Author: Jake Vanderplas  -- <vanderplas@astro.washington.edu>
# License: BSD 3 clause (C) 2011

import numpy as np
from ..base import BaseEstimator, TransformerMixin
from ..neighbors import NearestNeighbors, kneighbors_graph
from ..utils import check_array
from ..utils.graph import graph_shortest_path
from ..decomposition import KernelPCA
from ..preprocessing import KernelCenterer

class Isomap(BaseEstimator, TransformerMixin):
    """Isomap Embedding

    Non-linear dimensionality reduction through Isometric Mapping

    Read more in the :ref:`User Guide <isomap>`.

    Parameters
    ----------
    n_neighbors : integer
        number of neighbors to consider for each point.

    n_components : integer
        number of coordinates for the manifold

    eigen_solver : ['auto'|'arpack'|'dense']
        'auto' : Attempt to choose the most efficient solver
        for the given problem.

        'arpack' : Use Arnoldi decomposition to find the eigenvalues
        and eigenvectors.

        'dense' : Use a direct solver (i.e. LAPACK)
        for the eigenvalue decomposition.

    tol : float
        Convergence tolerance passed to arpack or lobpcg.
        not used if eigen_solver == 'dense'.

    max_iter : integer
        Maximum number of iterations for the arpack solver.
        not used if eigen_solver == 'dense'.

    path_method : string ['auto'|'FW'|'D']
        Method to use in finding shortest path.

        'auto' : attempt to choose the best algorithm automatically.

        'FW' : Floyd-Warshall algorithm.

        'D' : Dijkstra's algorithm.

    neighbors_algorithm : string ['auto'|'brute'|'kd_tree'|'ball_tree']
        Algorithm to use for nearest neighbors search,
        passed to neighbors.NearestNeighbors instance.

    Attributes
    ----------
    embedding_ : array-like, shape (n_samples, n_components)
        Stores the embedding vectors.

    kernel_pca_ : object
        `KernelPCA` object used to implement the embedding.

    training_data_ : array-like, shape (n_samples, n_features)
        Stores the training data.

    nbrs_ : sklearn.neighbors.NearestNeighbors instance
        Stores nearest neighbors instance, including BallTree or KDtree
        if applicable.

    dist_matrix_ : array-like, shape (n_samples, n_samples)
        Stores the geodesic distance matrix of training data.

    References
    ----------

    .. [1] Tenenbaum, J.B.; De Silva, V.; & Langford, J.C. A global geometric
           framework for nonlinear dimensionality reduction. Science 290 (5500)
    """

    def __init__(self, n_neighbors=5, n_components=2, eigen_solver='auto',
                 tol=0, max_iter=None, path_method='auto',
                 neighbors_algorithm='auto'):

        self.n_neighbors = n_neighbors
        self.n_components = n_components
        self.eigen_solver = eigen_solver
        self.tol = tol
        self.max_iter = max_iter
        self.path_method = path_method
        self.neighbors_algorithm = neighbors_algorithm
        self.nbrs_ = NearestNeighbors(n_neighbors=n_neighbors,
                                      algorithm=neighbors_algorithm)

    def _fit_transform(self, X):
        X = check_array(X)
        self.nbrs_.fit(X)
        self.training_data_ = self.nbrs_._fit_X
        self.kernel_pca_ = KernelPCA(n_components=self.n_components,
                                     kernel="precomputed",
                                     eigen_solver=self.eigen_solver,
                                     tol=self.tol, max_iter=self.max_iter)

        kng = kneighbors_graph(self.nbrs_, self.n_neighbors,
                               mode='distance')

        self.dist_matrix_ = graph_shortest_path(kng,
                                                method=self.path_method,
                                                directed=False)
        G = self.dist_matrix_ ** 2
        G *= -0.5

        self.embedding_ = self.kernel_pca_.fit_transform(G)

    def reconstruction_error(self):
        """Compute the reconstruction error for the embedding.

        Returns
        -------
        reconstruction_error : float

        Notes
        -------
        The cost function of an isomap embedding is

        ``E = frobenius_norm[K(D) - K(D_fit)] / n_samples``

        Where D is the matrix of distances for the input data X,
        D_fit is the matrix of distances for the output embedding X_fit,
        and K is the isomap kernel:

        ``K(D) = -0.5 * (I - 1/n_samples) * D^2 * (I - 1/n_samples)``
        """
        G = -0.5 * self.dist_matrix_ ** 2
        G_center = KernelCenterer().fit_transform(G)
        evals = self.kernel_pca_.lambdas_
        return np.sqrt(np.sum(G_center ** 2) - np.sum(evals ** 2)) / G.shape[0]

    def fit(self, X, y=None):
        """Compute the embedding vectors for data X

        Parameters
        ----------
        X : {array-like, sparse matrix, BallTree, KDTree, NearestNeighbors}
            Sample data, shape = (n_samples, n_features), in the form of a
            numpy array, precomputed tree, or NearestNeighbors
            object.

        Returns
        -------
        self : returns an instance of self.
        """
        self._fit_transform(X)
        return self

    def fit_transform(self, X, y=None):
        """Fit the model from data in X and transform X.

        Parameters
        ----------
        X: {array-like, sparse matrix, BallTree, KDTree}
            Training vector, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        self._fit_transform(X)
        return self.embedding_

    def transform(self, X):
        """Transform X.

        This is implemented by linking the points X into the graph of geodesic
        distances of the training data. First the `n_neighbors` nearest
        neighbors of X are found in the training data, and from these the
        shortest geodesic distances from each point in X to each point in
        the training data are computed in order to construct the kernel.
        The embedding of X is the projection of this kernel onto the
        embedding vectors of the training set.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        X = check_array(X)
        distances, indices = self.nbrs_.kneighbors(X, return_distance=True)

        #Create the graph of shortest distances from X to self.training_data_
        # via the nearest neighbors of X.
        #This can be done as a single array operation, but it potentially
        # takes a lot of memory.  To avoid that, use a loop:
        G_X = np.zeros((X.shape[0], self.training_data_.shape[0]))
        for i in range(X.shape[0]):
            G_X[i] = np.min((self.dist_matrix_[indices[i]]
                             + distances[i][:, None]), 0)

        G_X **= 2
        G_X *= -0.5

        return self.kernel_pca_.transform(G_X)


from dask.utils import raises
from dask.core import (istask, get, get_dependencies, flatten, subs,
        preorder_traversal)

def contains(a, b):
    """

    >>> contains({'x': 1, 'y': 2}, {'x': 1})
    True
    >>> contains({'x': 1, 'y': 2}, {'z': 3})
    False
    """
    return all(a.get(k) == v for k, v in b.items())

def inc(x):
    return x + 1

def add(x, y):
    return x + y

def test_istask():
    assert istask((inc, 1))
    assert not istask(1)
    assert not istask((1, 2))

d = {':x': 1,
     ':y': (inc, ':x'),
     ':z': (add, ':x', ':y')}

def test_preorder_traversal():
    t = (add, 1, 2)
    assert list(preorder_traversal(t)) == [add, 1, 2]
    t = (add, (add, 1, 2), (add, 3, 4))
    assert list(preorder_traversal(t)) == [add, add, 1, 2, add, 3, 4]
    t = (add, (sum, [1, 2]), 3)
    assert list(preorder_traversal(t)) == [add, sum, list, 1, 2, 3]

def test_get():
    assert get(d, ':x') == 1
    assert get(d, ':y') == 2
    assert get(d, ':z') == 3
    assert get(d, 'pass-through') == 'pass-through'

def test_memoized_get():
    try:
        import toolz
    except ImportError:
        return
    cache = dict()
    getm = toolz.memoize(get, cache=cache, key=lambda args, kwargs: args[1:])

    result = getm(d, ':z', get=getm)
    assert result == 3

    assert contains(cache, {(':x',): 1,
                            (':y',): 2,
                            (':z',): 3})

def test_data_not_in_dict_is_ok():
    d = {'x': 1, 'y': (add, 'x', 10)}
    assert get(d, 'y') == 11

def test_get_with_list():
    d = {'x': 1, 'y': 2, 'z': (sum, ['x', 'y'])}

    assert get(d, ['x', 'y']) == [1, 2]
    assert get(d, 'z') == 3

def test_get_with_nested_list():
    d = {'x': 1, 'y': 2, 'z': (sum, ['x', 'y'])}

    assert get(d, [['x'], 'y']) == [[1], 2]
    assert get(d, 'z') == 3

def test_get_works_with_unhashables_in_values():
    f = lambda x, y: x + len(y)
    d = {'x': 1, 'y': (f, 'x', set([1]))}

    assert get(d, 'y') == 2

def test_get_laziness():
    def isconcrete(arg):
        return isinstance(arg, list)

    d = {'x': 1, 'y': 2, 'z': (isconcrete, ['x', 'y'])}

    assert get(d, ['x', 'y']) == [1, 2]
    assert get(d, 'z') == False

def test_get_dependencies_nested():
    dsk = {'x': 1, 'y': 2,
           'z': (add, (inc, [['x']]), 'y')}

    assert get_dependencies(dsk, 'z') == set(['x', 'y'])

def test_get_dependencies_empty():
    dsk = {'x': (inc,)}
    assert get_dependencies(dsk, 'x') == set()

def test_nested_tasks():
    d = {'x': 1,
         'y': (inc, 'x'),
         'z': (add, (inc, 'x'), 'y')}

    assert get(d, 'z') == 4

def test_get_stack_limit():
    d = dict(('x%s' % (i+1), (inc, 'x%s' % i)) for i in range(10000))
    d['x0'] = 0
    assert get(d, 'x10000') == 10000
    # introduce cycle
    d['x5000'] = (inc, 'x5001')
    assert raises(RuntimeError, lambda: get(d, 'x10000'))
    assert get(d, 'x4999') == 4999

def test_flatten():
    assert list(flatten(())) == []
    assert list(flatten('foo')) == ['foo']

def test_subs():
    assert subs((sum, [1, 'x']), 'x', 2) == (sum, [1, 2])
    assert subs((sum, [1, ['x']]), 'x', 2) == (sum, [1, [2]])

def test_subs_with_unfriendly_eq():
    try:
        import numpy as np
    except:
        return
    else:
        task = (np.sum, np.array([1, 2]))
        assert (subs(task, (4, 5), 1) == task) is True

def test_subs_with_surprisingly_friendly_eq():
    try:
        import pandas as pd
    except:
        return
    else:
        df = pd.DataFrame()
        assert subs(df, 'x', 1) is df


"""Prepare a binned matrix of misalignments and plot it in different ways"""

import click
import pysam
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.path import Path
import matplotlib.patches as patches
from matplotlib.colors import LogNorm
import numpy as np

def we_have_too_many_bins(bins):
  return sum([len(bb) for bb in bins]) > 5000  # This is our threshold for too many bins to compute

def autoscale_bin_size(chrom_lens, bin_cnt=100.0):
  return int(sum(chrom_lens) / bin_cnt)

def compute_misalignment_matrix_from_bam(bam_fp, bin_size=None, i_know_what_i_am_doing=False):
  """Create a matrix of binned mis-alignments

  :param bam_fp: input BAM
  :param bin_size: size of bin in mega bases
  :param i_know_what_i_am_doing: Set this to override the runtime warning of too many bins

  """
  def binnify(_pos, _bins):
    for n in range(1, len(_bins)):
      if _pos < _bins[n]:
        return n - 1
    return len(_bins) - 1  # Should not get here

  chrom_lens = [hdr['LN'] for hdr in bam_fp.header['SQ']]
  bin_size = bin_size * 1e6 if bin_size is not None else autoscale_bin_size(chrom_lens)
  bins = [np.array(range(0, hdr['LN'], bin_size) + [hdr['LN']], dtype=int) for hdr in bam_fp.header['SQ']]
  if not i_know_what_i_am_doing and we_have_too_many_bins(bins):
    raise RuntimeWarning('The number of bins will be very large. '
                         'If you are sure you want to do this, '
                         'use the --i-know-what-i-am-doing flag.')

  bin_centers = [(bb[:-1] + bb[1:]) / 2.0 for bb in bins]
  # Rows = source (correct pos) Cols = destination (aligned pos)
  matrices = [[np.zeros(shape=(len(bins[j]) - 1, len(bins[i]) - 1), dtype='uint32') for i in range(len(bins))] for j in range(len(bins))]

  # TAG TYPE VALUE
  # XR  i    Aligned chromosome
  # XP  i    Aligned pos
  for r in bam_fp:
    c_chrom, c_pos, a_chrom, a_pos = r.reference_id, r.pos, r.get_tag('XR'), r.get_tag('XP')
    c_pos_binned, a_pos_binned = binnify(c_pos, bins[c_chrom]), binnify(a_pos, bins[a_chrom])
    matrices[c_chrom][a_chrom][c_pos_binned, a_pos_binned] += 1
  return chrom_lens, bins, bin_centers, matrices

def plot_genome_as_a_circle(ax, chrom_lens, chrom_gap=np.pi / 50, chrom_radius=1.0, chrom_thick=5, r_max=1.05):
  """Plot the chromosomes on a circle."""
  total_len = sum(chrom_lens)
  radians_per_base = (2.0 * np.pi - len(chrom_lens) * chrom_gap) / total_len  # With allowance for chrom gaps

  theta_stops, x_ticks, x_tick_labels = [], [], []
  delta_radian = 0.01
  start_radian = 0
  for ch_no, l in enumerate(chrom_lens):
    end_radian = start_radian + l * radians_per_base
    theta = np.arange(start_radian, end_radian, delta_radian)
    theta_stops.append((start_radian, end_radian))
    ax.plot(theta, [chrom_radius * 1.01] * theta.size, lw=chrom_thick, zorder=-1)  # , color=[.3, .3, .3])
    x_ticks.append((start_radian + end_radian)/2)
    x_tick_labels.append(str(ch_no + 1))
    start_radian = end_radian + chrom_gap

  plt.setp(ax.get_yticklabels(), visible=False)
  ax.grid(False)
  plt.setp(ax, xticks=x_ticks, xticklabels=x_tick_labels)
  ax.set_rmax(r_max)
  return theta_stops

def plot_read_mis_alignments_on_a_circle(ax, chrom_lens, bins, bin_centers, matrices, theta_stops,
                                         chrom_radius=1.0, scaling_factor=0.01):
  scaling_factor *= 0.01
  # http://matplotlib.org/users/path_tutorial.html
  codes = [
    Path.MOVETO,
    Path.CURVE4,
    Path.CURVE4,
    Path.CURVE4,
  ]
  for i in range(len(bins)):
    for j in range(len(bins)):
      mat = matrices[i][j]
      range_bp_origin, range_bp_dest = float(chrom_lens[i]), float(chrom_lens[j])
      offset_origin, offset_dest = theta_stops[i][0], theta_stops[j][0]
      range_origin, range_dest = theta_stops[i][1] - theta_stops[i][0], theta_stops[j][1] - theta_stops[j][0]
      scale_origin, scale_dest = range_origin / range_bp_origin, range_dest / range_bp_dest
      c_origin, c_dest = offset_origin + bin_centers[i] * scale_origin, offset_dest + bin_centers[j] * scale_dest
      this_origin, this_dest = np.tile(c_origin, c_dest.shape[0]), np.repeat(c_dest, c_origin.shape[0])
      mat_flat = mat.ravel()
      idx, = mat_flat.nonzero()
      for ii in idx:
        t0, t1 = this_origin[ii], this_dest[ii]

        this_radius = max(min(1.0, abs(t1 - t0) / np.pi), 0.05) * chrom_radius
        vertices = [
          (t0, chrom_radius),  # P0
          (t0, chrom_radius - this_radius),  # P1
          (t1, chrom_radius - this_radius),  # P2
          (t1, chrom_radius),  # P3
        ]
        path = Path(vertices, codes)
        patch = patches.PathPatch(path, facecolor='none', lw=scaling_factor * mat_flat[ii])
        ax.add_patch(patch)

def circle_plot(chrom_lens, bins, bin_centers, matrices, scaling_factor):
  """Plot the confusion matrix as a circle plot."""
  fig = plt.figure()
  ax = fig.add_subplot(111, polar=True)
  theta_stops = plot_genome_as_a_circle(ax, chrom_lens)
  plot_read_mis_alignments_on_a_circle(ax, chrom_lens, bins, bin_centers, matrices, theta_stops, chrom_radius=1.0, scaling_factor=scaling_factor)

def plot_genome_as_a_square(ax, bins, chrom_gap=1000, chrom_thick=5):
  """Plot the chromosomes on a matrix."""
  start_pos, linear_stops, x_ticks, x_tick_labels = chrom_gap, [], [], []
  for ch_no, b in enumerate(bins):
    linear_stops.append([start_pos, start_pos + b[-1]])
    ax.plot([x + start_pos for x in b], [0 for _ in b], color='k' if ch_no % 2 else 'gray', lw=chrom_thick, zorder=-1)
    ax.plot([0 for _ in b], [x + start_pos for x in b], color='k' if ch_no % 2 else 'gray', lw=chrom_thick, zorder=-1)
    x_ticks.append((start_pos + start_pos + b[-1]) / 2)
    x_tick_labels.append(str(ch_no + 1))
    start_pos += b[-1] + chrom_gap

  #plt.setp(ax.get_yticklabels(), visible=False)
  ax.grid(False)
  plt.setp(ax, xticks=x_ticks, xticklabels=x_tick_labels, yticks=x_ticks, yticklabels=x_tick_labels)
  return linear_stops

def plot_read_mis_alignments_as_a_matrix(ax, chrom_lens, bins, bin_centers, matrices, linear_stops,
                                         scaling_factor=1.0, plot_grid=True):
  for i in range(len(bins)):
    for j in range(len(bins)):
      mat = matrices[i][j]
      range_bp_x, range_bp_y = float(chrom_lens[i]), float(chrom_lens[j])
      offset_x, offset_y = linear_stops[i][0], linear_stops[j][0]
      range_x, range_y = linear_stops[i][1] - linear_stops[i][0], linear_stops[j][1] - linear_stops[j][0]
      scale_x, scale_y = range_x / range_bp_x, range_y / range_bp_y
      cx, cy = offset_x + bin_centers[i] * scale_x, offset_y + bin_centers[j] * scale_y
      this_x, this_y = np.tile(cx, cy.shape[0]), np.repeat(cy, cx.shape[0])
      if plot_grid: ax.plot(this_x, this_y, '.', color=(0.8, 0.8, 0.8), ms=2, zorder=-1)
      mat_flat = mat.ravel()
      idx, = mat_flat.nonzero()
      if idx.size > 0:
        ax.scatter(this_x[idx], this_y[idx], mat_flat[idx] * scaling_factor, facecolors='none')

def matrix_plot(chrom_lens, bins, bin_centers, matrices, scaling_factor, plot_grid=True):
  """Plot the confusion matrix as a ... matrix."""
  fig = plt.figure()
  ax = fig.add_subplot(111)
  linear_stops = plot_genome_as_a_square(ax, bins, chrom_gap=max(chrom_lens) * 0.1)
  plot_read_mis_alignments_as_a_matrix(ax, chrom_lens, bins, bin_centers, matrices, linear_stops,
                                       scaling_factor=scaling_factor, plot_grid=plot_grid)
  plt.setp(ax, aspect=1, xlabel='Correct', ylabel='Aligned')

def is_grid_too_dense(bins):
  return sum([len(bb) for bb in bins]) > 100  # This is our threshold for too dense a grid to show

def auto_scale_scaling_factor(matrices, scale=1000.0):
  return scale / max([matrices[i][j].max() for i in range(len(matrices)) for j in range(len(matrices[i]))])

@click.command()
@click.argument('badbam', type=click.Path(exists=True))
@click.option('--circle', type=click.Path(), help='Name of figure file for circle plot')
@click.option('--matrix', type=click.Path(), help='Name of figure file for matrix plot')
@click.option('--bin-size', type=float, default=None, help='Bin size in Mb. Omit to auto-scale')
@click.option('--scaling-factor', type=float, default=None, help='Scale size of disks/lines in plot. Omit to auto-scale')
@click.option('--i-know-what-i-am-doing', is_flag=True, help='Override bin density safety')
def cli(badbam, circle, matrix, bin_size, scaling_factor, i_know_what_i_am_doing):
  """Prepare a binned matrix of mis-alignments and plot it in different ways"""
  chrom_lens, bins, bin_centers, matrices = \
    compute_misalignment_matrix_from_bam(pysam.AlignmentFile(badbam, 'rb'),
                                         bin_size=bin_size, i_know_what_i_am_doing=i_know_what_i_am_doing)
  scaling_factor = scaling_factor or auto_scale_scaling_factor(matrices)

  if circle is not None:
    circle_plot(chrom_lens, bins, bin_centers, matrices, scaling_factor)
    plt.savefig(circle)
  if matrix is not None:
    matrix_plot(chrom_lens, bins, bin_centers, matrices, scaling_factor,
                plot_grid=not is_grid_too_dense(bins))
    plt.savefig(matrix)

if __name__ == '__main__':
  cli()


# USAGE
# python color_kmeans.py --image images/jp.png --clusters 3

# Author: Adrian Rosebrock
# Website: www.pyimagesearch.com

# import the necessary packages
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import numpy as np
import argparse
import cv2

def centroid_histogram(clt):
    # grab the number of different clusters and create a histogram
    # based on the number of pixels assigned to each cluster
    numLabels = np.arange(0, len(np.unique(clt.labels_)) + 1)
    (hist, _) = np.histogram(clt.labels_, bins = numLabels)

    # normalize the histogram, such that it sums to one
    hist = hist.astype("float")
    hist /= hist.sum()

    # return the histogram
    return hist

def plot_colors(hist, centroids):
    # initialize the bar chart representing the relative frequency
    # of each of the colors
    bar = np.zeros((50, 300, 3), dtype = "uint8")
    startX = 0

    # loop over the percentage of each cluster and the color of
    # each cluster
    for (percent, color) in zip(hist, centroids):
        # plot the relative percentage of each cluster
        endX = startX + (percent * 300)
        cv2.rectangle(bar, (int(startX), 0), (int(endX), 50),
                      color.astype("uint8").tolist(), -1)
        startX = endX

    # return the bar chart
    return bar

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True, help="Path to the image")
ap.add_argument("-c", "--clusters", required=True, type=int,
                help="# of clusters")
args = vars(ap.parse_args())

# load the image and convert it from BGR to RGB so that
# we can dispaly it with matplotlib
image = cv2.imread(args["image"])
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

# show our image
plt.figure()
plt.axis("off")
plt.imshow(image)

# reshape the image to be a list of pixels
image = image.reshape((image.shape[0]*image.shape[1], 3))

# cluster the pixel intensities
clt = KMeans(n_clusters=args["clusters"])
clt.fit(image)

# build a histogram of clusters and then create a figure
# representing the number of pixels labeled to each color
hist = centroid_histogram(clt)
bar = plot_colors(hist, clt.cluster_centers_)

# show our color bart
plt.figure()
plt.axis("off")
plt.imshow(bar)
plt.show()


from elasticsearch import Elasticsearch
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib.dates import AutoDateLocator, AutoDateFormatter
import numpy as np
import datetime as dt
import math
import json

with open('sites.json', 'r+') as txt:
    sitesArray = json.load(txt)

with open('cms.json', 'r+') as txt:
    cmsLocate = json.load(txt)

with open("config", "r+") as txt:
    contents = list(map(str.rstrip, txt))

def conAtlasTime(time):
    return (dt.datetime.strptime(time, '%Y-%m-%dT%X')).replace(tzinfo=dt.timezone.utc).timestamp()

def utcDate(time):
    return dt.datetime.fromtimestamp(time, dt.timezone.utc)

esAtlas = Elasticsearch([{
    'host': contents[2], 'port': contents[3]
}], timeout=50)

esHCC = Elasticsearch([{
    'host': contents[0], 'port': contents[1]
}], timeout=50)

scrollPreserve="3m"
startDate = "2016-07-17T00:00:00"
endDate = "2016-07-25T00:00:00"
tenMin = np.multiply(10,60)

loc = {}
loc["location"] = np.array([])

def atlasLatency(srcSite, destSite):
    queryAtlas={"query" :
                {"bool": {
                   "must": [
                     {"match" :
                         {"_type" : "latency" }
                     },
                     {"match" :
                         {"srcSite" : srcSite }
                     },
                     {"match" :
                         {"destSite" : destSite }
                     },
                     {"range" : {
                         "timestamp" : {
                             #"gt" : int(conAtlasTime(startDate)),
                             #"lt" : int(conAtlasTime(endDate))
                             "gt" : startDate,
                             "lt" : endDate
                         }
                     }}
                   ]
                 }
                }
               }
    scannerAtlas = esAtlas.search(index="network_weather_2-*",
                                  body=queryAtlas,
                                  search_type="scan",
                                  scroll=scrollPreserve)
    scrollIdAtlas = scannerAtlas['_scroll_id']
    atlasTotalRec = scannerAtlas["hits"]["total"]
    arrRet = np.array([])

    if atlasTotalRec == 0:
        return None
    else:
        while atlasTotalRec > 0:
            responseAtlas = esAtlas.scroll(scroll_id=scrollIdAtlas,
                                       scroll=scrollPreserve)
            for hit in responseAtlas["hits"]["hits"]:
                tempRay = None # Initialize
                if hit["_source"]["src"] == hit["_source"]["MA"]: # means MA is the src site
                    tempRay = np.array([#hit["_source"]["timestamp"],
                                        #hit["_source"]["timestamp"]
                                        conAtlasTime(hit["_source"]["timestamp"]),
                                        conAtlasTime(hit["_source"]["timestamp"])
                                          - np.multiply(4, np.multiply(60, 60)),
                                        float(hit["_source"]["delay_mean"]),
                                        float(hit["_source"]["delay_median"]),
                                        float(hit["_source"]["delay_sd"]),
                                        float(0.0)])
                elif hit["_source"]["dest"] == hit["_source"]["MA"]: # means MA is the dest site
                    tempRay = np.array([#hit["_source"]["timestamp"],
                                        #hit["_source"]["timestamp"]
                                        conAtlasTime(hit["_source"]["timestamp"]),
                                        conAtlasTime(hit["_source"]["timestamp"])
                                          - np.multiply(4, np.multiply(60, 60)),
                                        float(hit["_source"]["delay_mean"]),
                                        float(hit["_source"]["delay_median"]),
                                        float(hit["_source"]["delay_sd"]),
                                        float(1.0)])
                else:
                    raise NameError('MA is not the src or dest')
                if arrRet.size == 0:
                    arrRet = np.reshape(tempRay, (1,6))
                else:
                    arrRet = np.vstack((arrRet, tempRay))
            atlasTotalRec -= len(responseAtlas['hits']['hits'])
        arrRet.view('f8,f8,f8,f8,f8,f8').sort(order=['f0'], axis=0)
        return arrRet

def atlasPacketLoss(srcSite, destSite):
    queryAtlas={"query" :
                {"bool": {
                   "must": [
                     {"match" :
                         {"_type" : "packet_loss_rate"}
                     },
                     {"match" :
                         {"srcSite" : srcSite }
                     },
                     {"match" :
                         {"destSite" : destSite }
                     },
                     {"range" : {
                         "timestamp" : {
                             #"gt" : int(conAtlasTime(startDate)),
                             #"lt" : int(conAtlasTime(endDate))
                             "gt" : startDate,
                             "lt" : endDate
                         }
                     }}
                   ]
                 }
                }
               }
    scannerAtlas = esAtlas.search(index="network_weather_2-*",
                                  body=queryAtlas,
                                  search_type="scan",
                                  scroll=scrollPreserve)
    scrollIdAtlas = scannerAtlas['_scroll_id']
    atlasTotalRec = scannerAtlas["hits"]["total"]
    arrRet = np.array([])

    if atlasTotalRec == 0:
        return None
    else:
        while atlasTotalRec > 0:
            responseAtlas = esAtlas.scroll(scroll_id=scrollIdAtlas,
                                           scroll=scrollPreserve)
            for hit in responseAtlas["hits"]["hits"]:
                tempRay = None # Initialize
                if hit["_source"]["src"] == hit["_source"]["MA"]: # means MA is the src site
                    tempRay = np.array([#hit["_source"]["timestamp"],
                                        #hit["_source"]["timestamp"]
                                        conAtlasTime(hit["_source"]["timestamp"]),
                                        conAtlasTime(hit["_source"]["timestamp"])
                                          - np.multiply(4, np.multiply(60, 60)),
                                        float(hit["_source"]["packet_loss"]),
                                        float(0.0)])
                elif hit["_source"]["dest"] == hit["_source"]["MA"]: # means MA is the dest site
                    tempRay = np.array([#hit["_source"]["timestamp"],
                                        #hit["_source"]["timestamp"]
                                        conAtlasTime(hit["_source"]["timestamp"]),
                                        conAtlasTime(hit["_source"]["timestamp"])
                                          - np.multiply(4, np.multiply(60, 60)),
                                        float(hit["_source"]["packet_loss"]),
                                        float(1.0)])
                else:
                    raise NameError('MA is not src or dest')
                if arrRet.size == 0:
                    arrRet = np.reshape(tempRay, (1, 4))
                else:
                    arrRet = np.vstack((arrRet, tempRay))

            atlasTotalRec -= len(responseAtlas['hits']['hits'])
        arrRet.view('f8,f8,f8,f8').sort(order=['f0'], axis=0)
        return arrRet

def atlasThroughput(srcSite, destSite):
    queryAtlas={"query" :
                {"bool": {
                   "must": [
                     {"match" :
                         {"_type" : "throughput"}
                     },
                     {"match" :
                         {"srcSite" : srcSite }
                     },
                     {"match" :
                         {"destSite" : destSite }
                     },
                     {"range" : {
                         "timestamp" : {
                             #"gt" : int(conAtlasTime(startDate)),
                             #"lt" : int(conAtlasTime(endDate))
                             "gt" : startDate,
                             "lt" : endDate
                         }
                     }}
                   ]
                 }
                }
               }
    scannerAtlas = esAtlas.search(index="network_weather_2-*",
                                  body=queryAtlas,
                                  search_type="scan",
                                  scroll=scrollPreserve)
    scrollIdAtlas = scannerAtlas['_scroll_id']
    atlasTotalRec = scannerAtlas["hits"]["total"]
    arrRet = np.array([])
    if atlasTotalRec == 0:
        return None
    else:
        while atlasTotalRec > 0:
            responseAtlas = esAtlas.scroll(scroll_id=scrollIdAtlas,
                                           scroll=scrollPreserve)
            for hit in responseAtlas["hits"]["hits"]:
                tempRay = None #Initialize in local context
                if hit["_source"]["src"] == hit["_source"]["MA"]: # Means MA is the src site
                    tempRay = np.array([#hit["_source"]["timestamp"],
                                        #hit["_source"]["timestamp"]
                                        conAtlasTime(hit["_source"]["timestamp"]),
                                        conAtlasTime(hit["_source"]["timestamp"])
                                          - np.multiply(4, np.multiply(60, 60)),
                                        float(hit["_source"]["throughput"]),
                                        float(0.0)])
                elif hit["_source"]["dest"] == hit["_source"]["MA"]: #Means MA is the dest site
                    tempRay = np.array([#hit["_source"]["timestamp"],
                                        #hit["_source"]["timestamp"]
                                        conAtlasTime(hit["_source"]["timestamp"]),
                                        conAtlasTime(hit["_source"]["timestamp"])
                                          - np.multiply(4, np.multiply(60, 60)),
                                        float(hit["_source"]["throughput"]),
                                        float(1.0)])
                else:
                    raise NameError('MA is not src or dest')
                if arrRet.size == 0:
                    arrRet = np.reshape(tempRay, (1, 4))
                else:
                    arrRet = np.vstack((arrRet, tempRay))
            atlasTotalRec -= len(responseAtlas['hits']['hits'])
        arrRet.view('f8,f8,f8,f8').sort(order=['f0'], axis=0)
        return arrRet

def hccQuery(site):
    queryHCC={"query" :
              {"bool": {
                  "must": [
                      {"match" :
                         {"CMS_JobType" : "Processing"}
                      },
                      {"range" :
                         {"EventRate" : {"gte" : "0"}}
                      },
                      #{"match" :
                      #   {"TaskType" : "Production"}
                      #},
                      {"range" : {
                         "CompletionDate" : {
                             "gt" : int(conAtlasTime(startDate)),
                             "lt" : int(conAtlasTime(endDate))
                         }
                      }},
                      {"match" :
                         {"DataLocationsCount" : 1}
                      },
                      {"match" :
                         {"Site" : site }
                      },
                      {"match" :
                         {"InputData" : "Offsite"}
                      }
                  ]
              }
          }
      }

    scannerHCC = esHCC.search(index="cms-*",
                              doc_type="job",
                              body=queryHCC,
                              search_type="scan",
                              scroll=scrollPreserve)
    scrollIdHCC = scannerHCC['_scroll_id']
    countHitsHCC = scannerHCC["hits"]["total"]
    arrRet = {}
    if countHitsHCC == 0:
        return None
    else:
        while countHitsHCC > 0:
            responseHCC = esHCC.scroll(scroll_id=scrollIdHCC,
                                       scroll=scrollPreserve)
            for hit in responseHCC["hits"]["hits"]:
                location = hit["_source"]["DataLocations"]
                if str(location[0]).lower() in cmsLocate["locations"]:
                    tempHolder = np.array([hit["_source"]["CpuEff"],
                                           #hit["_source"]["EventRate"],
                                           hit["_source"]["ChirpCMSSWEventRate"],
                                           hit["_source"]["JobCurrentStartDate"],
                                           hit["_source"]["JobFinishedHookDone"],
                                           hit["_source"]["CpuTimeHr"],
                                           hit["_source"]["WallClockHr"],
                                           hit["_source"]["RequestCpus"],
                                           hit["_source"]["MemoryMB"],
                                           hit["_source"]["QueueHrs"],
                                           hit["_source"]["RequestMemory"],
                                           hit["_source"]["CoreHr"],
                                           hit["_source"]["CpuBadput"],
                                           hit["_source"]["KEvents"]])
                    if not str(location[0]) in loc["location"]:
                        loc["location"] = np.append(loc["location"],
                                                    str(location[0]))
                        arrRet[str(location[0])] = np.reshape(tempHolder, (1,13))
                    else:
                        arrRet[str(location[0])] = np.vstack((arrRet[str(location[0])],tempHolder))

            countHitsHCC -= len(responseHCC['hits']['hits'])
        for hit in arrRet:
            #print(arrRet)
            #tempRay = arrRet[str(hit)]
            #arrRet[str(hit)] = tempRay[tempRay[:,2].argsort()]
            #arrRet[str(hit)] = sorted(arrRet[str(hit)], key=lambda x : x[2])
            arrRet[str(hit)].view('f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8').sort(order=['f2'], axis=0)

        return arrRet

with PdfPages('CMS_Plots.pdf') as pp:
    d = pp.infodict()
    d['Title'] = 'CMS Grid Plots'
    d['Author'] = u'Jerrod T. Dixon\xe4nen'
    d['Subject'] = 'Plot of network affects on grid jobs'
    d['Keywords'] = 'PdfPages matplotlib CMS grid'
    d['CreationDate'] = dt.datetime.today()
    d['ModDate'] = dt.datetime.today()

    for hit in cmsLocate["locations"]:
        loc["location"] = np.array([])
        hccResult = hccQuery(hit)
        for note in loc["location"]:
            atlasT = None #atlasThroughput(sitesArray[hit], sitesArray[note.lower()])
            atlasP = atlasPacketLoss(sitesArray[hit], sitesArray[note.lower()])
            atlasL = atlasLatency(sitesArray[hit], sitesArray[note.lower()])

            tempArr = hccResult[note]
            arrCpu = np.array([]);
            arrEvent = np.array([]);
            arrStart = np.array([]);
            arrEnd = np.array([]);
            for tpl in tempArr:
                arrCpu = np.append(arrCpu, tpl[0]);
                arrEvent = np.append(arrEvent, tpl[1]);
                arrStart = np.append(arrStart, utcDate(tpl[2]));
                arrEnd = np.append(arrEnd, utcDate(tpl[3]));

            figH, axH = plt.subplots(2, sharex=True)
            axH[1].xaxis.set_major_formatter(AutoDateFormatter(locator=AutoDateLocator(),
                                                              defaultfmt="%m-%d %H:%M"))
            figH.autofmt_xdate(bottom=0.2, rotation=30, ha='right')
            axH[0].plot(arrStart, arrCpu, 'bs')
            axH[0].hlines(arrCpu,
                          arrStart,
                          arrEnd)
            axH[0].set_ylabel("CpuEff")
            axH[0].set_title(str("2016 From " + hit + " To " + note))

            axH[1].plot(arrStart, arrEvent, 'bs')
            axH[1].hlines(arrEvent,
                          arrStart,
                          arrEnd)
            axH[1].set_ylabel("EventRate")
            pp.savefig(figH)
            plt.close(figH)
            #axA[2].xaxis.set_major_formatter(AutoDateFormatter(locator=AutoDateLocator(),
            #                                                   defaultfmt="%m-%d %H:%M"))
            if not type(atlasP) == type(None):
                #tDate = np.array([])
                #tDatef = np.array([])
                #tPut = np.array([])
                pDate = np.array([])
                pDatef = np.array([])
                pLoss = np.array([])
                #for tpl in atlasT:
                #    tDate = np.append(tDate, tpl[0])
                #    tDatef = np.append(tDatef, tpl[1])
                #    tPut = np.append(tPut, tpl[2])
                for tpl in atlasP:
                    pDate = np.append(pDate, tpl[0])
                    pDatef = np.append(pDatef, tpl[1])
                    pLoss = np.append(pLoss, tpl[2])
                figA, axA = plt.subplots(2, sharex=True)
                axA[0].set_title(str("2016 From " + \
                                     hit + " (" + \
                                     sitesArray[hit] + \
                                     ")" + " To " + \
                                     note + " (" + sitesArray[note.lower()] + ")"))
                figA.autofmt_xdate(bottom=0.2, rotation=30, ha='right')
                axA[0].plot(pDate, pLoss, 'bs')
                axA[0].set_ylabel("Packet Loss")
                axA[0].hlines(pLoss,
                              pDatef,
                              pDate)

                #axA[1].set_ylabel("Throughput")
                #axA[1].plot(tDate, tPut, 'bs')
                #axA[1].hlines(tPut,
                #              tDatef,
                #              tDate)
                pp.savefig(figA)
                plt.close(figA)

            if not type(atlasL) == type(None):
                lDate = np.array([])
                lDatef = np.array([])
                lMean = np.array([])
                lMedian = np.array([])
                lStd = np.array([])
                for tpl in atlasL:
                    lDate = np.append(lDate, tpl[0])
                    lDatef = np.append(lDatef, tpl[1])
                    lMean = np.append(lMean, tpl[2])
                    lMedian = np.append(lMedian, tpl[3])
                    lStd = np.append(lStd, tpl[4])
                figL, axL = plt.subplots(3, sharex=True)
                axL[0].set_title(str("2016 Latency From " + \
                                     hit + " (" + \
                                     sitesArray[hit] + \
                                     ")" + " To " + \
                                     note + " (" + sitesArray[note.lower()] + ")"))
                figL.autofmt_xdate(bottom=0.2, rotation=30, ha='right')
                axL[0].set_ylabel("Mean")
                axL[0].plot(lDate, lMean, 'bs', label="delay_mean")
                axL[0].hlines(lMean,
                              lDatef,
                              lDate)
                axL[1].set_ylabel("Median")
                axL[1].plot(lDate, lMedian, 'rs', label="delay_median")
                axL[1].hlines(lMedian,
                              lDatef,
                              lDate)
                axL[2].set_ylabel("Std. Dev")
                axL[2].plot(lDate, lStd, 'g^', label="delay_sd")
                axL[2].hlines(lStd,
                              lDatef,
                              lDate)
                pp.savefig(figL)
                plt.close(figL)


# Copyright (C) 2010-2019 The ESPResSo project
#
# This file is part of ESPResSo.
#
# ESPResSo is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ESPResSo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
Simulate the flow of a lattice-Boltzmann fluid past a cylinder,
obtain the velocity profile in polar coordinates and compare it
to the analytical solution.
"""
import numpy as np
import matplotlib.pyplot as plt

import espressomd

required_features = ["CUDA", "LB_BOUNDARIES_GPU"]
espressomd.assert_features(required_features)

import espressomd.lb
import espressomd.observables
import espressomd.shapes
import espressomd.lbboundaries
import espressomd.accumulators

system = espressomd.System(box_l=[10.0, 10.0, 5.0])
system.time_step = 0.01
system.cell_system.skin = 0.4
n_steps = 500

lb_fluid = espressomd.lb.LBFluidGPU(
    agrid=1.0, dens=1.0, visc=1.0, tau=0.01, ext_force_density=[0, 0, 0.15], kT=1.0, seed=32)
system.actors.add(lb_fluid)
system.thermostat.set_lb(LB_fluid=lb_fluid, seed=23)
fluid_obs = espressomd.observables.CylindricalLBVelocityProfile(
    center=[5.0, 5.0, 0.0],
    axis='z',
    n_r_bins=100,
    n_phi_bins=1,
    n_z_bins=1,
    min_r=0.0,
    max_r=4.0,
    min_phi=-np.pi,
    max_phi=np.pi,
    min_z=0.0,
    max_z=10.0,
    sampling_delta_x=0.05,
    sampling_delta_y=0.05,
    sampling_delta_z=1.0)
cylinder_shape = espressomd.shapes.Cylinder(
    center=[5.0, 5.0, 5.0],
    axis=[0, 0, 1],
    direction=-1,
    radius=4.0,
    length=20.0)
cylinder_boundary = espressomd.lbboundaries.LBBoundary(shape=cylinder_shape)
system.lbboundaries.add(cylinder_boundary)
system.integrator.run(n_steps)

accumulator = espressomd.accumulators.MeanVarianceCalculator(obs=fluid_obs)
system.auto_update_accumulators.add(accumulator)
system.integrator.run(n_steps)

lb_fluid_profile = accumulator.get_mean()
lb_fluid_profile = np.reshape(lb_fluid_profile, (100, 1, 1, 3))

def poiseuille_flow(r, R, ext_force_density):
    return ext_force_density * 1. / 4 * (R**2.0 - r**2.0)

# Please note that due to symmetry and interpolation, a plateau is seen
# near r=0.
n_bins = len(lb_fluid_profile[:, 0, 0, 2])
r_max = 4.0
r = np.linspace(0.0, r_max, n_bins)
plt.plot(r, lb_fluid_profile[:, 0, 0, 2], label='LB profile')
plt.plot(r, poiseuille_flow(r, r_max, 0.15), label='analytical solution')
plt.legend()
plt.show()


#!/usr/bin/env python
"""
=================================
Path with L1- Logistic Regression
=================================

Computes path on IRIS dataset.

"""
print(__doc__)

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model
from sklearn import datasets
from sklearn.svm import l1_min_c

iris = datasets.load_iris()
X = iris.data
y = iris.target

X = X[y != 2]
y = y[y != 2]

X -= np.mean(X, 0)

###############################################################################
# Demo path functions

cs = l1_min_c(X, y, loss='log') * np.logspace(0, 3)

print("Computing regularization path ...")
start = datetime.now()
clf = linear_model.LogisticRegression(C=1.0, penalty='l1', tol=1e-6)
coefs_ = []
for c in cs:
    clf.set_params(C=c)
    clf.fit(X, y)
    coefs_.append(clf.coef_.ravel().copy())
print("This took ", datetime.now() - start)

coefs_ = np.array(coefs_)
plt.plot(np.log10(cs), coefs_)
ymin, ymax = plt.ylim()
plt.xlabel('log(C)')
plt.ylabel('Coefficients')
plt.title('Logistic Regression Path')
plt.axis('tight')
plt.show()