#!/usr/bin/env python
# -*- coding:utf-8 -*-
#
# written by Shotaro Fujimoto
# 2016-12-16

import matplotlib.pyplot as plt
# from mpl_toolkits.mplot3d.axes3d import Axes3D
import matplotlib.cm as cm
import numpy as np
import set_data_path

class Visualizer(object):
    def __init__(self, subjects):
        self.data_path_list = set_data_path.data_path
        if len(subjects) != 0:
            for subject in subjects:
                getattr(self, 'result_' + subject)()

    def load_data(self, _path):
        data = np.load(_path)
        beta = data['beta']
        try:
            size_dist_ave = data['size_dist_ave']
            if len(size_dist_ave) == 0:
                raise KeyError
            return self.load_data_averaged(_path)
        except KeyError:
            pass

        num_of_strings = data['num_of_strings']
        frames = data['frames']
        Ls = data['Ls'].astype(np.float)
        # Ls = (3 * Ls * (Ls + 1) + 1)
        size_dist = data['size_dist']

        N0 = np.array([l[1] for l in size_dist], dtype=np.float) / num_of_strings
        n0 = N0[1:]
        S = np.array([np.sum(l) for l in size_dist], dtype=np.float) / num_of_strings
        n1 = (S[1:] - n0) * 2.

        N = []
        for l in size_dist:
            dot = np.dot(np.arange(len(l)), np.array(l).T)
            N.append(dot)
        # N = np.array([np.dot(np.arange(len(l)), np.array(l).T) for l in size_dist])
        N_all = 3. * Ls * (Ls + 1.) + 1
        N = np.array(N, dtype=np.float) / num_of_strings
        N_minus = N_all - N
        N_minus_rate = N_minus / N_all
        n_minus = N_minus[1:] - N_minus[:-1]
        n1_ave = n1 / np.sum(n1)
        n2 = (6 * Ls[1:]) - (n0 + n1 + n_minus)

        self.beta = beta
        self.num_of_strings = num_of_strings
        self.frames = frames
        self.Ls = Ls
        self.N = N
        self.N_minus = N_minus
        self.N_minus_rate = N_minus_rate
        self.S = S
        self.n0 = n0
        self.n1 = n1
        self.n2 = n2
        self.n_minus = n_minus
        self.n1_ave = n1_ave

    def load_data_averaged(self, _path):
        data = np.load(_path)
        beta = data['beta']
        num_of_strings = data['num_of_strings']
        frames = data['frames']
        Ls = data['Ls'].astype(np.float)
        # Ls = (3 * Ls * (Ls + 1) + 1)
        # size_dist = data['size_dist']
        size_dist_ave = data['size_dist_ave']

        N0 = np.array([l[1] for l in size_dist_ave], dtype=np.float)
        n0 = N0[1:]
        S = np.array([np.sum(l) for l in size_dist_ave], dtype=np.float)
        n1 = (S[1:] - n0) * 2.

        N = []
        for l in size_dist_ave:
            dot = np.dot(np.arange(len(l)), np.array(l).T)
            N.append(dot)
        # N = np.array([np.dot(np.arange(len(l)), np.array(l).T) for l in size_dist_ave])
        N_all = 3. * Ls * (Ls + 1.) + 1
        N = np.array(N, dtype=np.float)
        N_minus = N_all - N
        N_minus_rate = N_minus / N_all
        n_minus = N_minus[1:] - N_minus[:-1]
        n1_ave = n1 / np.sum(n1)
        n2 = (6 * Ls[1:]) - (n0 + n1 + n_minus)

        self.beta = beta
        self.num_of_strings = num_of_strings
        self.frames = frames
        self.Ls = Ls
        self.N = N
        self.N_all = N_all
        self.N_minus = N_minus
        self.N_minus_rate = N_minus_rate
        self.S = S
        self.n_all = 6 * Ls[1:]
        self.n0 = n0
        self.n1 = n1
        self.n2 = n2
        self.n_minus = n_minus
        self.n1_ave = n1_ave

    def result_N(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            ax.plot(self.Ls[1:], self.N[1:], '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_title('Occupied points in the cutting region' +
                    ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$N$')
        plt.show()

    def result_N_minus_rate(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            ax.plot(self.Ls[1:], self.N_minus_rate[1:], '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_title('The rate of not occupied site in all N' +
                    ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$N_{-1} / N_{\mathrm{all}}$')
        plt.show()

    def result_n0(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            ax.plot(self.Ls[1:], self.n0, '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_title('Averaged number of the sites which is the only member of \
                     a subcluster on the cutting edges.' +
                    ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$n_{0}$')
        plt.show()

    def result_n1(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            ax.plot(self.Ls[1:], self.n1, '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_title('Averaged number of the sites which is connected to a \
                    existing subcluster on the cutting edges.' +
                    ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$n_{1}$')
        plt.show()

    def result_n2(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            ax.plot(self.Ls[1:], self.n2, '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_title('Averaged number of the sites on the cutting edges which \
                    is connected to two neighbors.' +
                    ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$n_{2}$')
        plt.show()

    def result_n_minus(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            ax.plot(self.Ls[1:], self.n_minus, '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_title('Averaged number of the sites which is not occupied on \
                     the cutting edges.' +
                    ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$n_{-1}$')
        plt.show()

    def result_S(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            ax.plot(self.Ls[1:], self.S[1:] / np.sum(self.S[1:]), '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_ylim([0, ax.get_ylim()[1]])
        ax.set_title('Averaged number of the subclusters in the cutted region.'
                     + ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$S$')
        plt.show()

    def result_S_rate(self):
        fig, ax = plt.subplots()
        for i, result_data_path in enumerate(self.data_path_list):
            self.load_data(result_data_path)
            # ax.plot(self.Ls[1:], self.S[1:] / np.sum(self.S[1:]), '.',
            # ax.plot(self.Ls[1:], self.S[1:] / self.n_all, '.',
            ax.plot(self.Ls[1:], self.S[1:] / self.N[1:], '.',
                    label=r'$\beta = %2.2f$' % self.beta,
                    color=cm.viridis(float(i) / len(self.data_path_list)))
        ax.legend(loc='best')
        ax.set_ylim([0, ax.get_ylim()[1]])
        ax.set_title('Averaged number of the subclusters in the cutted region'
                     + ' (normalized)'
                     + ' (sample: {})'.format(self.num_of_strings))
        ax.set_xlabel(r'Cutting size $L$')
        ax.set_ylabel(r'$S$')
        plt.show()

if __name__ == '__main__':
    # subject: 'N', 'N_minus_rate', 'n0', 'n1', 'n2', 'n_minus', 'S'
    main = Visualizer(
        [
            # 'N',
            # 'N_minus_rate',
            # 'n0',
            # 'n1',
            # 'n2',
            # 'n_minus',
            'S',
            # 'S_rate'
        ]
    )


#!/usr/bin/env python

"""
Example RIFF (WAV contents) Data Parser

Sample data is written to a CSV file for analysis.
If matplotlib and numpy are available, signal plots (DFTs) are generated.
"""

import math
import os
import struct
import wave

try:
    import matplotlib.pyplot as plot
    import numpy
    import numpy.fft as fft
except ImportError:
    numeric_packages = False
else:
    numeric_packages = True

#=============================================================================
def frame2mag( frame ):
    ( i, q ) = struct.unpack( '<BB', frame )
    return math.sqrt( ( i ** 2 ) + ( q ** 2 ) )

#=============================================================================
def main( argv ):
    """ Script execution entry point """

    # check usage
    if len( argv ) < 2:
        print 'You must specify at least an input file.'
        return 0

    # start and length
    start = 0
    length = 1024
    if len( argv ) > 2:
        start = int( argv[ 2 ] )
    if len( argv ) > 3:
        length = int( argv[ 3 ] )

    # open file using wave module
    wfile = wave.open( argv[ 1 ], 'rb' )

    # print file info
    print 'Channels: %d\nSample width: %d\nFrame rate: %d\nFrames: %d' % (
        wfile.getnchannels(),
        wfile.getsampwidth(),
        wfile.getframerate(),
        wfile.getnframes()
    )

    # check for starting offset
    if start > 0:
        junk = wfile.readframes( start )

    # read frames
    frames = wfile.readframes( length )
    samples = []
    for i in range( length ):
        index = i * 2
        samples.append( frame2mag( frames[ index : ( index + 2 ) ] ) )

    # close wave file
    wfile.close()

    # plot
    if numeric_packages == True:
        fft_data = fft.fft( samples[ : 1024 ] )
        mags = numpy.absolute( fft_data )
        mags_db = [ 20 * numpy.log10( mag ) for mag in mags ]
        plot.figure( 1 )
        plot.plot( samples )
        plot.figure( 2 )
        plot.plot( mags_db )
        plot.show()

    # output
    oname = argv[ 1 ].replace( '.wav', '.csv' )
    ofile = open( oname, 'wb' )
    for sample in samples:
        ofile.write( '%d\n' % sample )
    ofile.close()

    # Return success.
    return 0

#=============================================================================
if __name__ == "__main__":
    import sys
    sys.exit( main( sys.argv ) )


# -*- coding: utf-8 -*-
#
# Yade documentation build configuration file, created by
# sphinx-quickstart on Mon Nov 16 21:49:34 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# relevant posts to sphinx ML
# http://groups.google.com/group/sphinx-dev/browse_thread/thread/b4fbc8d31d230fc4
# http://groups.google.com/group/sphinx-dev/browse_thread/thread/118598245d5f479b

#####################
## custom yade roles
#####################
##
## http://docutils.sourceforge.net/docs/howto/rst-roles.html

import sys, os, re
from docutils import nodes
from sphinx import addnodes
from sphinx.roles import XRefRole
import docutils

#
# needed for creating hyperlink targets.
# it should be cleand up and unified for both LaTeX and HTML via
# the pending_xref node which gets resolved to real link target
# by sphinx automatically once all docs have been processed.
#
# xrefs: http://groups.google.com/group/sphinx-dev/browse_thread/thread/d719d19307654548
#
#
import __builtin__
if 'latex' in sys.argv: __builtin__.writer='latex'
elif 'html' in sys.argv: __builtin__.writer='html'
elif 'epub' in sys.argv: __builtin__.writer='epub'
else: raise RuntimeError("Must have either 'latex' or 'html' on the command line (hack for reference styles)")

def yaderef_role(role,rawtext,text,lineno,inliner,options={},content=[]):
	"Handle the :yref:`` role, by making hyperlink to yade.wrapper.*. It supports :yref:`Link text<link target>` syntax, like usual hyperlinking roles."
	id=rawtext.split(':',2)[2][1:-1]
	txt=id; explicitText=False
	m=re.match('(.*)\s*<(.*)>\s*',id)
	if m:
		explicitText=True
		txt,id=m.group(1),m.group(2)
	id=id.replace('::','.')
	#node=nodes.reference(rawtext,docutils.utils.unescape(txt),refuri='http://beta.arcig.cz/~eudoxos/yade/doxygen/?search=%s'%id,**options)
	#node=nodes.reference(rawtext,docutils.utils.unescape(txt),refuri='yade.wrapper.html#yade.wrapper.%s'%id,**options)
	return [mkYrefNode(id,txt,rawtext,role,explicitText,lineno,options)],[]

def yadesrc_role(role,rawtext,lineno,inliner,options={},content=[]):
	"Handle the :ysrc:`` role, making hyperlink to git repository webpage with that path. Supports :ysrc:`Link text<file/name>` syntax, like usual hyperlinking roles. If target ends with ``/``, it is assumed to be a directory."
	id=rawtext.split(':',2)[2][1:-1]
	txt=id
	m=re.match('(.*)\s*<(.*)>\s*',id)
	if m:
		txt,id=m.group(1),m.group(2)
	return [nodes.reference(rawtext,docutils.utils.unescape(txt),refuri='https://github.com/yade/trunk/blob/master/%s'%id)],[] ### **options should be passed to nodes.reference as well

# map modules to their html (rst) filenames. Used for sub-modules, where e.g. SpherePack is yade._packSphere.SpherePack, but is documented from yade.pack.rst
moduleMap={
	'yade._packPredicates':'yade.pack',
	'yade._packSpheres':'yade.pack',
	'yade._packObb':'yade.pack'
}

class YadeXRefRole(XRefRole):
	#def process_link
	def process_link(self, env, refnode, has_explicit_title, title, target):
		print 'TARGET:','yade.wrapper.'+target
		return '[['+title+']]','yade.wrapper.'+target

def mkYrefNode(target,text,rawtext,role,explicitText,lineno,options={}):
	"""Create hyperlink to yade target. Targets starting with literal 'yade.' are absolute, but the leading 'yade.' will be stripped from the link text. Absolute tergets are supposed to live in page named yade.[module].html, anchored at #yade.[module2].[rest of target], where [module2] is identical to [module], unless mapped over by moduleMap.

	Other targets are supposed to live in yade.wrapper (such as c++ classes)."""

	writer=__builtin__.writer # to make sure not shadowed by a local var
	import string
	if target.startswith('yade.'):
		module='.'.join(target.split('.')[0:2])
		module2=(module if module not in moduleMap.keys() else moduleMap[module])
		if target==module: target='' # to reference the module itself
		uri=('%%%s#%s'%(module2,target) if writer=='latex' else '%s.html#%s'%(module2,target))
		if not explicitText and module!=module2:
			text=module2+'.'+'.'.join(target.split('.')[2:])
		text=string.replace(text,'yade.','',1)
	elif target.startswith('external:'):
		exttarget=target.split(':',1)[1]
		if not explicitText: text=exttarget
		target=exttarget if '.' in exttarget else 'module-'+exttarget
		uri=(('%%external#%s'%target) if writer=='latex' else 'external.html#%s'%target)
	else:
		uri=(('%%yade.wrapper#yade.wrapper.%s'%target) if writer=='latex' else 'yade.wrapper.html#yade.wrapper.%s'%target)
		#print writer,uri
	if 0:
		refnode=addnodes.pending_xref(rawtext,reftype=role,refexplicit=explicitText,reftarget=target)
		#refnode.line=lineno
		#refnode+=nodes.literal(rawtext,text,classes=['ref',role])
		return [refnode],[]
		#ret.rawtext,reftype=role,
	else:
		return nodes.reference(rawtext,docutils.utils.unescape(text),refuri=uri,**options)
	#return [refnode],[]

def ydefault_role(role,rawtext,text,lineno,inliner,options={},content=[]):
	"Handle the :ydefault:`something` role. fixSignature handles it now in the member signature itself, this merely expands to nothing."
	return [],[]
def yattrtype_role(role,rawtext,text,lineno,inliner,options={},content=[]):
	"Handle the :yattrtype:`something` role. fixSignature handles it now in the member signature itself, this merely expands to nothing."
	return [],[]
# FIXME: should return readable representation of bits of the number (yade.wrapper.AttrFlags enum)
def yattrflags_role(role,rawtext,text,lineno,inliner,options={},content=[]):
	"Handle the :yattrflags:`something` role. fixSignature handles it now in the member signature itself."
	return [],[]

from docutils.parsers.rst import roles
def yaderef_role_2(type,rawtext,text,lineno,inliner,options={},content=[]): return YadeXRefRole()('yref',rawtext,text,lineno,inliner,options,content)
roles.register_canonical_role('yref', yaderef_role)
roles.register_canonical_role('ysrc', yadesrc_role)
roles.register_canonical_role('ydefault', ydefault_role)
roles.register_canonical_role('yattrtype', yattrtype_role)
roles.register_canonical_role('yattrflags', yattrflags_role)

## http://sphinx.pocoo.org/config.html#confval-rst_epilog
rst_epilog = """

.. |yupdate| replace:: *(auto-updated)*
.. |ycomp| replace:: *(auto-computed)*
.. |ystatic| replace:: *(static)*
"""

import collections
def customExclude(app, what, name, obj, skip, options):
	if name=='clone':
		if 'Serializable.clone' in str(obj): return False
		return True
	#escape crash on non iterable __doc__ in some qt object
	if hasattr(obj,'__doc__') and obj.__doc__ and not isinstance(obj.__doc__, collections.Iterable): return True
	if hasattr(obj,'__doc__') and obj.__doc__ and ('|ydeprecated|' in obj.__doc__ or '|yhidden|' in obj.__doc__): return True
	#if re.match(r'\b(__init__|__reduce__|__repr__|__str__)\b',name): return True
	if name.startswith('_'):
		if name=='__init__':
			# skip boost classes with parameterless ctor (arg1=implicit self)
			if obj.__doc__=="\n__init__( (object)arg1) -> None": return True
			# skip undocumented ctors
			if not obj.__doc__: return True
			# skip default ctor for serializable, taking dict of attrs
			if obj.__doc__=='\n__init__( (object)arg1) -> None\n\nobject __init__(tuple args, dict kwds)': return True
			#for i,l in enumerate(obj.__doc__.split('\n')): print name,i,l,'##'
			return False
		return True
	return False

def isBoostFunc(what,obj):
	return what=='function' and obj.__repr__().startswith('<Boost.Python.function object at 0x')

def isBoostMethod(what,obj):
	"I don't know how to distinguish boost and non-boost methods..."
	return what=='method' and obj.__repr__().startswith('<unbound method ');

def replaceLaTeX(s):
	# replace single non-escaped dollars $...$ by :math:`...`
	# then \$ by single $
	s=re.sub(r'(?<!\\)\$([^\$]+)(?<!\\)\$',r'\ :math:`\1`\ ',s)
	return re.sub(r'\\\$',r'$',s)

def fixSrc(app,docname,source):
	source[0]=replaceLaTeX(source[0])

def fixDocstring(app,what,name,obj,options,lines):
	# remove empty default roles, which is not properly interpreted by docutils parser
	for i in range(0,len(lines)):
		lines[i]=lines[i].replace(':ydefault:``','')
		lines[i]=lines[i].replace(':yattrtype:``','')
		lines[i]=lines[i].replace(':yattrflags:``','')
		#lines[i]=re.sub(':``',':` `',lines[i])
	# remove signature of boost::python function docstring, which is the first line of the docstring
	if isBoostFunc(what,obj):
		l2=boostFuncSignature(name,obj)[1]
		# we must replace lines one by one (in-place) :-|
		# knowing that l2 is always shorter than lines (l2 is docstring with the signature stripped off)
		for i in range(0,len(lines)):
			lines[i]=l2[i] if i<len(l2) else ''
	elif isBoostMethod(what,obj):
		l2=boostFuncSignature(name,obj)[1]
		for i in range(0,len(lines)):
			lines[i]=l2[i] if i<len(l2) else ''
	# LaTeX: replace $...$ by :math:`...`
	# must be done after calling boostFuncSignature which uses original docstring
	for i in range(0,len(lines)): lines[i]=replaceLaTeX(lines[i])

def boostFuncSignature(name,obj,removeSelf=False):
	"""Scan docstring of obj, returning tuple of properly formatted boost python signature
	(first line of the docstring) and the rest of docstring (as list of lines).
	The rest of docstring is stripped of 4 leading spaces which are automatically
	added by boost.

	removeSelf will attempt to remove the first argument from the signature.
	"""
	doc=obj.__doc__
	if doc==None: # not a boost method
		return None,None
	nname=name.split('.')[-1]
	docc=doc.split('\n')
	if len(docc)<2: return None,docc
	doc1=docc[1]
	# functions with weird docstring, likely not documented by boost
	if not re.match('^'+nname+r'(.*)->.*$',doc1):
		return None,docc
	if doc1.endswith(':'): doc1=doc1[:-1]
	strippedDoc=doc.split('\n')[2:]
	# check if all lines are padded
	allLinesHave4LeadingSpaces=True
	for l in strippedDoc:
		if l.startswith('    '): continue
		allLinesHave4LeadingSpaces=False; break
	# remove the padding if so
	if allLinesHave4LeadingSpaces: strippedDoc=[l[4:] for l in strippedDoc]
	for i in range(len(strippedDoc)):
		# fix signatures inside docstring (one function with multiple signatures)
		strippedDoc[i],n=re.subn(r'([a-zA-Z_][a-zA-Z0-9_]*\() \(object\)arg1(, |)',r'\1',strippedDoc[i].replace('->','→'))
	# inspect dosctring after mangling
	if 'getViscoelasticFromSpheresInteraction' in name and False:
		print name
		print strippedDoc
		print '======================'
		for l in strippedDoc: print l
		print '======================'
	sig=doc1.split('(',1)[1]
	if removeSelf:
		# remove up to the first comma; if no comma present, then the method takes no arguments
		# if [ precedes the comma, add it to the result (ugly!)
		try:
			ss=sig.split(',',1)
			if ss[0].endswith('['): sig='['+ss[1]
			else: sig=ss[1]
		except IndexError:
			# grab the return value
			try:
				sig=') -> '+sig.split('->')[-1]
		#if 'Serializable' in name: print 1000*'#',name
			except IndexError:
				sig=')'
	return '('+sig,strippedDoc

def fixSignature(app, what, name, obj, options, signature, return_annotation):
	#print what,name,obj,signature#,dir(obj)
	if what=='attribute':
		doc=unicode(obj.__doc__)
		ret=''
		m=re.match('.*:ydefault:`(.*?)`.*',doc)
		if m:
			typ=''
			#try:
			#	clss='.'.join(name.split('.')[:-1])
			#	instance=eval(clss+'()')
			#	typ='; '+getattr(instance,name.split('.')[-1]).__class__.__name__
			#	if typ=='; NoneType': typ=''
			#except TypeError: ##no registered converted
			#	typ=''
			dfl=m.group(1)
			m2=re.match(r'\s*\(\s*\(\s*void\s*\)\s*\"(.*)\"\s*,\s*(.*)\s*\)\s*',dfl)
			if m2: dfl="%s, %s"%(m2.group(2),m2.group(1))
			if dfl!='': ret+=' (='+dfl+'%s)'%typ
			else: ret+=' (=uninitalized%s)'%typ
		#m=re.match('.*\[(.{,8})\].*',doc)
		#m=re.match('.*:yunit:`(.?*)`.*',doc)
		#if m:
		#	units=m.group(1)
		#	print '@@@@@@@@@@@@@@@@@@@@@',name,units
		#	ret+=' ['+units+']'
		return ret,None
	elif what=='class':
		ret=[]
		if len(obj.__bases__)>0:
			base=obj.__bases__[0]
			while base.__module__!='Boost.Python':
				ret+=[base.__name__]
				if len(base.__bases__)>0: base=base.__bases__[0]
				else: break
		if len(ret):
			return ' (inherits '+u' → '.join(ret)+')',None
		else: return None,None
	elif isBoostFunc(what,obj):
		sig=boostFuncSignature(name,obj)[0] or ' (wrapped c++ function)'
		return sig,None
	elif isBoostMethod(what,obj):
		sig=boostFuncSignature(name,obj,removeSelf=True)[0]
		return sig,None
	#else: print what,name,obj.__repr__()
	#return None,None

from sphinx import addnodes
def parse_ystaticattr(env,attr,attrnode):
	m=re.match(r'([a-zA-Z0-9_]+)\.(.*)\(=(.*)\)',attr)
	if not m:
		print 100*'@'+' Static attribute %s not matched'%attr
		attrnode+=addnodes.desc_name(attr,attr)
	klass,name,default=m.groups()
	#attrnode+=addnodes.desc_type('static','static')
	attrnode+=addnodes.desc_name(name,name)
	plist=addnodes.desc_parameterlist()
	if default=='': default='unspecified'
	plist+=addnodes.desc_parameter('='+default,'='+default)
	attrnode+=plist
	attrnode+=addnodes.desc_annotation('  [static]','  [static]')
	return klass+'.'+name

#############################
## set tab size
###################
## http://groups.google.com/group/sphinx-dev/browse_thread/thread/35b8071ffe9a8feb
def setup(app):
	from sphinx.highlighting import lexers
	from pygments.lexers.compiled import CppLexer
	lexers['cpp'] = CppLexer(tabsize=3)
	lexers['c++'] = CppLexer(tabsize=3)
	from pygments.lexers.agile import PythonLexer
	lexers['python'] = PythonLexer(tabsize=3)

	app.connect('source-read',fixSrc)

	app.connect('autodoc-skip-member',customExclude)
	app.connect('autodoc-process-signature',fixSignature)
	app.connect('autodoc-process-docstring',fixDocstring)
	app.add_description_unit('ystaticattr',None,objname='static attribute',indextemplate='pair: %s; static method',parse_node=parse_ystaticattr)

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.append(os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.

#
# HACK: change ipython console regexp from ipython_console_highlighting.py
import re
sys.path.append(os.path.abspath('.'))

import yade.config

if 1:
	if yade.runtime.ipython_version<12:
		import ipython_directive as id
	else:
		if 12<=yade.runtime.ipython_version<13:
			import ipython_directive012 as id
                elif 13<=yade.runtime.ipython_version<200:
			import ipython_directive013 as id
                else:
			import ipython_directive200 as id

	#The next four lines are for compatibility with IPython 0.13.1
	ipython_rgxin =re.compile(r'(?:In |Yade )\[(\d+)\]:\s?(.*)\s*')
	ipython_rgxout=re.compile(r'(?:Out| ->  )\[(\d+)\]:\s?(.*)\s*')
	ipython_promptin ='Yade [%d]:'
	ipython_promptout=' ->  [%d]: '
	ipython_cont_spaces='     '
	#For IPython <=0.12, the following lines are used
	id.rgxin =re.compile(r'(?:In |Yade )\[(\d+)\]:\s?(.*)\s*')
	id.rgxout=re.compile(r'(?:Out| ->  )\[(\d+)\]:\s?(.*)\s*')
	id.rgxcont=re.compile(r'(?:   +)\.\.+:\s?(.*)\s*')
	id.fmtin  ='Yade [%d]:'
	id.fmtout =' ->  [%d]: '  # for some reason, out and cont must have the trailing space
	id.fmtcont='     .\D.: '
	id.rc_override=dict(prompt_in1="Yade [\#]:",prompt_in2="     .\D.:",prompt_out=r" ->  [\#]: ")
	if yade.runtime.ipython_version<12:
		id.reconfig_shell()

	import ipython_console_highlighting as ich
	ich.IPythonConsoleLexer.input_prompt = re.compile("(Yade \[[0-9]+\]: )")
	ich.IPythonConsoleLexer.output_prompt = re.compile("(( ->  |Out)|\[[0-9]+\]: )")
	ich.IPythonConsoleLexer.continue_prompt = re.compile("\s+\.\.\.+:")

extensions = [
		'sphinx.ext.autodoc',
		'sphinx.ext.autosummary',
		'sphinx.ext.coverage',
		'sphinx.ext.pngmath',
		'sphinx.ext.graphviz',
		'sphinx.ext.viewcode',
		'sphinx.ext.inheritance_diagram',
		'matplotlib.sphinxext.plot_directive',
		'matplotlib.sphinxext.only_directives',
		#'matplotlib.sphinxext.mathmpl',
		'ipython_console_highlighting',
		'youtube',
		'sphinx.ext.todo',
		]

if yade.runtime.ipython_version<12:
	extensions.append('ipython_directive')
else:
	if 12<=yade.runtime.ipython_version<13:
		extensions.append('ipython_directive012')
        elif 13<=yade.runtime.ipython_version<200:
		extensions.append('ipython_directive013')
        else:
		extensions.append('ipython_directive200')

# the sidebar extension
if False:
	if writer=='html':
		extensions+=['sphinx.ext.sidebar']

	sidebar_all=True
	sidebar_relling=True
	#sidebar_abbrev=True
	sidebar_tocdepth=3

## http://trac.sagemath.org/sage_trac/attachment/ticket/7549/trac_7549-doc_inheritance_underscore.patch
# GraphViz includes dot, neato, twopi, circo, fdp.
graphviz_dot = 'dot'
inheritance_graph_attrs = { 'rankdir' : 'BT' }
inheritance_node_attrs = { 'height' : 0.5, 'fontsize' : 12, 'shape' : 'oval' }
inheritance_edge_attrs = {}

my_latex_preamble=r'''
\usepackage{euler} % must be loaded before fontspec for the whole doc (below); this must be kept for pngmath, however
\usepackage{hyperref}
\usepackage{amsmath}
\usepackage{amsbsy}
%\usepackage{mathabx}
\usepackage{underscore}
\usepackage[all]{xy}

% Metadata of the pdf output
\hypersetup{pdftitle={Yade Documentation}}
\hypersetup{pdfauthor={V. Smilauer, E. Catalano, B. Chareyre, S. Dorofeenko, J. Duriez, A. Gladky, J. Kozicki, C. Modenese, L. Scholtes, L. Sibille, J. Stransky, K. Thoeni}}

% symbols
\let\mat\boldsymbol % matrix
\let\vec\boldsymbol % vector
\let\tens\boldsymbol % tensor

\def\normalized#1{\widehat{#1}}
\def\locframe#1{\widetilde{#1}}

% timestep
\def\Dt{\Delta t}
\def\Dtcr{\Dt_{\rm cr}}

% algorithm complexity
\def\bigO#1{\ensuremath{\mathcal{O}(#1)}}

% variants for greek symbols
\let\epsilon\varepsilon
\let\theta\vartheta
\let\phi\varphi

% shorthands
\let\sig\sigma
\let\eps\epsilon

% variables at different points of time
\def\prev#1{#1^-}
\def\pprev#1{#1^\ominus}
\def\curr#1{#1^{\circ}}
\def\nnext#1{#1^\oplus}
\def\next#1{#1^+}

% shorthands for geometry
\def\currn{\curr{\vec{n}}}
\def\currC{\curr{\vec{C}}}
\def\uT{\vec{u}_T}
\def\curruT{\curr{\vec{u}}_T}
\def\prevuT{\prev{\vec{u}}_T}
\def\currn{\curr{\vec{n}}}
\def\prevn{\prev{\vec{n}}}

% motion
\def\pprevvel{\pprev{\dot{\vec{u}}}}
\def\nnextvel{\nnext{\dot{\vec{u}}}}
\def\curraccel{\curr{\ddot{\vec{u}}}}
\def\prevpos{\prev{\vec{u}}}
\def\currpos{\curr{\vec{u}}}
\def\nextpos{\next{\vec{u}}}
\def\curraaccel{\curr{\dot{\vec{\omega}}}}
\def\pprevangvel{\pprev{\vec{\omega}}}
\def\nnextangvel{\nnext{\vec{\omega}}}
\def\loccurr#1{\curr{\locframe{#1}}}

\def\numCPU{n_{\rm cpu}}
\DeclareMathOperator{\Align}{Align}
\DeclareMathOperator{\sign}{sgn}

% sorting algorithms
\def\isleq#1{\currelem{#1}\ar@/^/[ll]^{\leq}}
\def\isnleq#1{\currelem{#1}\ar@/^/[ll]^{\not\leq}}
\def\currelem#1{\fbox{$#1$}}
\def\sortSep{||}
\def\sortInv{\hbox{\phantom{||}}}
\def\sortlines#1{\xymatrix@=3pt{#1}}
\def\crossBound{||\mkern-18mu<}

'''

pngmath_latex_preamble=r'\usepackage[active]{preview}'+my_latex_preamble

pngmath_use_preview=True

# Add any paths that contain templates here, relative to this directory.
templates_path = ['templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index-toctree'

# General information about the project.
project = u'Yade'
copyright = u'2009, Václav Šmilauer'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = yade.config.version
# The full version, including alpha/beta/rc tags.
release = yade.config.revision

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
modindex_common_prefix = ['yade.']

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {'stickysidebar':'true','collapsiblesidebar':'true','rightsidebar':'false'}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = 'fig/yade-logo.png'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = 'fig/yade-favicon.ico'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['static-html']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

html_index='index.html'

# Additional templates that should be rendered to pages, maps page names to
# template names.
html_additional_pages = { 'index':'index.html'}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'Yadedoc'

# -- Options for LaTeX output --------------------------------------------------

my_maketitle=r'''
\begin{titlepage}

\begin{flushright}
\hrule{}

% Upper part of the page
\begin{flushright}
\includegraphics[width=0.15\textwidth]{yade-logo.png}\par
\end{flushright}
\vspace{20 mm}
\text{\sffamily\bfseries\Huge Yade Documentation}\\
\vspace{5 mm}
\vspace{70 mm}
\begin{sffamily}\bfseries\Large
V\'{a}clav \v{S}milauer, Emanuele Catalano, Bruno Chareyre, Sergei Dorofeenko, Jerome Duriez, Anton Gladky, Janek Kozicki, Chiara Modenese, Luc Scholt\`{e}s, Luc Sibille, Jan Str\'{a}nsk\'{y}, Klaus Thoeni
\end{sffamily}
\vspace{20 mm}
\hrule{}

\vfill
% Bottom of the page
\textit{\Large Release '''\
+yade.config.revision\
+r''', \today}
\end{flushright}

\end{titlepage}

\text{\sffamily\bfseries\LARGE Authors}\\
\\
\text{\sffamily\bfseries\Large V\'{a}clav \v{S}milauer}\\
\text{\sffamily\Large Freelance consultant (http://woodem.eu)}\\
\\
\text{\sffamily\bfseries\Large Emanuele Catalano}\\
\text{\sffamily\Large Grenoble INP, UJF, CNRS, lab. 3SR}\\
\\
\text{\sffamily\bfseries\Large Bruno Chareyre}\\
\text{\sffamily\Large Grenoble INP, UJF, CNRS, lab. 3SR}\\
\\
\text{\sffamily\bfseries\Large Sergei Dorofeenko}\\
\text{\sffamily\Large IPCP RAS, Chernogolovka}\\
\\
\text{\sffamily\bfseries\Large Jerome Duriez}\\
\text{\sffamily\Large Grenoble INP, UJF, CNRS, lab. 3SR}\\
\\
\text{\sffamily\bfseries\Large Anton Gladky}\\
\text{\sffamily\Large TU Bergakademie Freiberg}\\
\\
\text{\sffamily\bfseries\Large Janek Kozicki}\\
\text{\sffamily\Large Gdansk University of Technology - lab. 3SR Grenoble University }\\
\\
\text{\sffamily\bfseries\Large Chiara Modenese}\\
\text{\sffamily\Large University of Oxford}\\
\\
\text{\sffamily\bfseries\Large Luc Scholt\`{e}s}\\
\text{\sffamily\Large Grenoble INP, UJF, CNRS, lab. 3SR}\\
\\
\text{\sffamily\bfseries\Large Luc Sibille}\\
\text{\sffamily\Large University of Nantes, lab. GeM}\\
\\
\text{\sffamily\bfseries\Large Jan Str\'{a}nsk\'{y}}\\
\text{\sffamily\Large CVUT Prague}\\
\\
\text{\sffamily\bfseries\Large Klaus Thoeni}
\text{\sffamily\Large The University of Newcastle (Australia)}\\

\text{\sffamily\bfseries\large Citing this document}\\
In order to let users cite Yade consistently in publications, we provide a list of bibliographic references for the different parts of the documentation. This way of acknowledging Yade is also a way to make developments and documentation of Yade more attractive for researchers, who are evaluated on the basis of citations of their work by others. We therefore kindly ask users to cite Yade as accurately as possible in their papers, as explained in http://yade-dem/doc/citing.html.

'''

latex_elements=dict(
	papersize='a4paper',
	fontpkg=r'''
\usepackage{euler}
\usepackage{fontspec,xunicode,xltxtra}
%\setmainfont[BoldFont={LMRoman10 Bold}]{CMU Concrete} %% CMU Concrete must be installed by hand as otf
	''',
	utf8extra='',
	fncychap='',
	preamble=my_latex_preamble,
	footer='',
	inputenc='',
	fontenc='',
	maketitle=my_maketitle,
)

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index-toctree', 'Yade.tex', u'Yade Documentation',
   u'Václav Šmilauer', 'manual'),
   ('index-toctree_manuals', 'YadeManuals.tex', u'Yade Tutorial and Manuals',
   u'Václav Šmilauer', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
latex_logo = 'fig/yade-logo.png'

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True


import warnings

DISPLAY_WIDTH = "display_width"
ARITHMETIC_JOIN = "arithmetic_join"
ENABLE_CFTIMEINDEX = "enable_cftimeindex"
FILE_CACHE_MAXSIZE = "file_cache_maxsize"
WARN_FOR_UNCLOSED_FILES = "warn_for_unclosed_files"
CMAP_SEQUENTIAL = "cmap_sequential"
CMAP_DIVERGENT = "cmap_divergent"
KEEP_ATTRS = "keep_attrs"
DISPLAY_STYLE = "display_style"

OPTIONS = {
    DISPLAY_WIDTH: 80,
    ARITHMETIC_JOIN: "inner",
    ENABLE_CFTIMEINDEX: True,
    FILE_CACHE_MAXSIZE: 128,
    WARN_FOR_UNCLOSED_FILES: False,
    CMAP_SEQUENTIAL: "viridis",
    CMAP_DIVERGENT: "RdBu_r",
    KEEP_ATTRS: "default",
    DISPLAY_STYLE: "html",
}

_JOIN_OPTIONS = frozenset(["inner", "outer", "left", "right", "exact"])
_DISPLAY_OPTIONS = frozenset(["text", "html"])

def _positive_integer(value):
    return isinstance(value, int) and value > 0

_VALIDATORS = {
    DISPLAY_WIDTH: _positive_integer,
    ARITHMETIC_JOIN: _JOIN_OPTIONS.__contains__,
    ENABLE_CFTIMEINDEX: lambda value: isinstance(value, bool),
    FILE_CACHE_MAXSIZE: _positive_integer,
    WARN_FOR_UNCLOSED_FILES: lambda value: isinstance(value, bool),
    KEEP_ATTRS: lambda choice: choice in [True, False, "default"],
    DISPLAY_STYLE: _DISPLAY_OPTIONS.__contains__,
}

def _set_file_cache_maxsize(value):
    from ..backends.file_manager import FILE_CACHE

    FILE_CACHE.maxsize = value

def _warn_on_setting_enable_cftimeindex(enable_cftimeindex):
    warnings.warn(
        "The enable_cftimeindex option is now a no-op "
        "and will be removed in a future version of xarray.",
        FutureWarning,
    )

_SETTERS = {
    FILE_CACHE_MAXSIZE: _set_file_cache_maxsize,
    ENABLE_CFTIMEINDEX: _warn_on_setting_enable_cftimeindex,
}

def _get_keep_attrs(default):
    global_choice = OPTIONS["keep_attrs"]

    if global_choice == "default":
        return default
    elif global_choice in [True, False]:
        return global_choice
    else:
        raise ValueError(
            "The global option keep_attrs must be one of" " True, False or 'default'."
        )

class set_options:
    """Set options for xarray in a controlled context.

    Currently supported options:

    - ``display_width``: maximum display width for ``repr`` on xarray objects.
      Default: ``80``.
    - ``arithmetic_join``: DataArray/Dataset alignment in binary operations.
      Default: ``'inner'``.
    - ``file_cache_maxsize``: maximum number of open files to hold in xarray's
      global least-recently-usage cached. This should be smaller than your
      system's per-process file descriptor limit, e.g., ``ulimit -n`` on Linux.
      Default: 128.
    - ``warn_for_unclosed_files``: whether or not to issue a warning when
      unclosed files are deallocated (default False). This is mostly useful
      for debugging.
    - ``cmap_sequential``: colormap to use for nondivergent data plots.
      Default: ``viridis``. If string, must be matplotlib built-in colormap.
      Can also be a Colormap object (e.g. mpl.cm.magma)
    - ``cmap_divergent``: colormap to use for divergent data plots.
      Default: ``RdBu_r``. If string, must be matplotlib built-in colormap.
      Can also be a Colormap object (e.g. mpl.cm.magma)
    - ``keep_attrs``: rule for whether to keep attributes on xarray
      Datasets/dataarrays after operations. Either ``True`` to always keep
      attrs, ``False`` to always discard them, or ``'default'`` to use original
      logic that attrs should only be kept in unambiguous circumstances.
      Default: ``'default'``.
    - ``display_style``: display style to use in jupyter for xarray objects.
      Default: ``'text'``. Other options are ``'html'``.

    You can use ``set_options`` either as a context manager:

    >>> ds = xr.Dataset({"x": np.arange(1000)})
    >>> with xr.set_options(display_width=40):
    ...     print(ds)
    <xarray.Dataset>
    Dimensions:  (x: 1000)
    Coordinates:
      * x        (x) int64 0 1 2 3 4 5 6 ...
    Data variables:
        *empty*

    Or to set global options:

    >>> xr.set_options(display_width=80)
    """

    def __init__(self, **kwargs):
        self.old = {}
        for k, v in kwargs.items():
            if k not in OPTIONS:
                raise ValueError(
                    "argument name %r is not in the set of valid options %r"
                    % (k, set(OPTIONS))
                )
            if k in _VALIDATORS and not _VALIDATORS[k](v):
                if k == ARITHMETIC_JOIN:
                    expected = f"Expected one of {_JOIN_OPTIONS!r}"
                elif k == DISPLAY_STYLE:
                    expected = f"Expected one of {_DISPLAY_OPTIONS!r}"
                else:
                    expected = ""
                raise ValueError(
                    f"option {k!r} given an invalid value: {v!r}. " + expected
                )
            self.old[k] = OPTIONS[k]
        self._apply_update(kwargs)

    def _apply_update(self, options_dict):
        for k, v in options_dict.items():
            if k in _SETTERS:
                _SETTERS[k](v)
        OPTIONS.update(options_dict)

    def __enter__(self):
        return

    def __exit__(self, type, value, traceback):
        self._apply_update(self.old)


#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import pandas as pd
from pandas.api.types import CategoricalDtype

from pyspark import pandas as ps
from pyspark.pandas.config import option_context
from pyspark.pandas.tests.data_type_ops.testing_utils import TestCasesUtils
from pyspark.testing.pandasutils import PandasOnSparkTestCase

class BinaryOpsTest(PandasOnSparkTestCase, TestCasesUtils):
    @property
    def pser(self):
        return pd.Series([b"1", b"2", b"3"])

    @property
    def psser(self):
        return ps.from_pandas(self.pser)

    def test_add(self):
        psser = self.psser
        pser = self.pser
        self.assert_eq(psser + b"1", pser + b"1")
        self.assert_eq(psser + psser, pser + pser)
        self.assert_eq(psser + psser.astype("bytes"), pser + pser.astype("bytes"))
        self.assertRaises(TypeError, lambda: psser + "x")
        self.assertRaises(TypeError, lambda: psser + 1)

        with option_context("compute.ops_on_diff_frames", True):
            for psser in self.pssers:
                self.assertRaises(TypeError, lambda: self.psser + psser)
            self.assert_eq(self.psser + self.psser, self.pser + self.pser)

    def test_sub(self):
        self.assertRaises(TypeError, lambda: self.psser - "x")
        self.assertRaises(TypeError, lambda: self.psser - 1)

        with option_context("compute.ops_on_diff_frames", True):
            for psser in self.pssers:
                self.assertRaises(TypeError, lambda: self.psser - psser)

    def test_mul(self):
        self.assertRaises(TypeError, lambda: self.psser * "x")
        self.assertRaises(TypeError, lambda: self.psser * 1)

        with option_context("compute.ops_on_diff_frames", True):
            for psser in self.pssers:
                self.assertRaises(TypeError, lambda: self.psser * psser)

    def test_truediv(self):
        self.assertRaises(TypeError, lambda: self.psser / "x")
        self.assertRaises(TypeError, lambda: self.psser / 1)

        with option_context("compute.ops_on_diff_frames", True):
            for psser in self.pssers:
                self.assertRaises(TypeError, lambda: self.psser / psser)

    def test_floordiv(self):
        self.assertRaises(TypeError, lambda: self.psser // "x")
        self.assertRaises(TypeError, lambda: self.psser // 1)

        with option_context("compute.ops_on_diff_frames", True):
            for psser in self.pssers:
                self.assertRaises(TypeError, lambda: self.psser // psser)

    def test_mod(self):
        self.assertRaises(TypeError, lambda: self.psser % "x")
        self.assertRaises(TypeError, lambda: self.psser % 1)

        with option_context("compute.ops_on_diff_frames", True):
            for psser in self.pssers:
                self.assertRaises(TypeError, lambda: self.psser % psser)

    def test_pow(self):
        self.assertRaises(TypeError, lambda: self.psser ** "x")
        self.assertRaises(TypeError, lambda: self.psser ** 1)

        with option_context("compute.ops_on_diff_frames", True):
            for psser in self.pssers:
                self.assertRaises(TypeError, lambda: self.psser ** psser)

    def test_radd(self):
        self.assert_eq(b"1" + self.psser, b"1" + self.pser)
        self.assertRaises(TypeError, lambda: "x" + self.psser)
        self.assertRaises(TypeError, lambda: 1 + self.psser)

    def test_rsub(self):
        self.assertRaises(TypeError, lambda: "x" - self.psser)
        self.assertRaises(TypeError, lambda: 1 - self.psser)

    def test_rmul(self):
        self.assertRaises(TypeError, lambda: "x" * self.psser)
        self.assertRaises(TypeError, lambda: 2 * self.psser)

    def test_rtruediv(self):
        self.assertRaises(TypeError, lambda: "x" / self.psser)
        self.assertRaises(TypeError, lambda: 1 / self.psser)

    def test_rfloordiv(self):
        self.assertRaises(TypeError, lambda: "x" // self.psser)
        self.assertRaises(TypeError, lambda: 1 // self.psser)

    def test_rmod(self):
        self.assertRaises(TypeError, lambda: 1 % self.psser)

    def test_rpow(self):
        self.assertRaises(TypeError, lambda: "x" ** self.psser)
        self.assertRaises(TypeError, lambda: 1 ** self.psser)

    def test_and(self):
        self.assertRaises(TypeError, lambda: self.psser & True)
        self.assertRaises(TypeError, lambda: self.psser & False)
        self.assertRaises(TypeError, lambda: self.psser & self.psser)

    def test_rand(self):
        self.assertRaises(TypeError, lambda: True & self.psser)
        self.assertRaises(TypeError, lambda: False & self.psser)

    def test_or(self):
        self.assertRaises(TypeError, lambda: self.psser | True)
        self.assertRaises(TypeError, lambda: self.psser | False)
        self.assertRaises(TypeError, lambda: self.psser | self.psser)

    def test_ror(self):
        self.assertRaises(TypeError, lambda: True | self.psser)
        self.assertRaises(TypeError, lambda: False | self.psser)

    def test_from_to_pandas(self):
        data = [b"1", b"2", b"3"]
        pser = pd.Series(data)
        psser = ps.Series(data)
        self.assert_eq(pser, psser.to_pandas())
        self.assert_eq(ps.from_pandas(pser), psser)

    def test_astype(self):
        pser = self.pser
        psser = self.psser
        self.assert_eq(pd.Series(["1", "2", "3"]), psser.astype(str))
        self.assert_eq(pser.astype("category"), psser.astype("category"))
        cat_type = CategoricalDtype(categories=[b"2", b"3", b"1"])
        self.assert_eq(pser.astype(cat_type), psser.astype(cat_type))

if __name__ == "__main__":
    import unittest
    from pyspark.pandas.tests.data_type_ops.test_binary_ops import *  # noqa: F401

    try:
        import xmlrunner  # type: ignore[import]

        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
    except ImportError:
        testRunner = None
    unittest.main(testRunner=testRunner, verbosity=2)


# -*- coding: utf-8 -*-
#
# hsmmlearn documentation build configuration file, created by
# sphinx-quickstart on Fri Jan  1 17:33:24 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# Avoid using C libraries on RTD
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if on_rtd:
    from mock import Mock as MagicMock

    class Mock(MagicMock):
        @classmethod
        def __getattr__(cls, name):
            return MagicMock()

    MOCK_MODULES = ['numpy', 'scipy', 'scipy.stats', 'matplotlib',
                    'matplotlib.pyplot', 'hsmmlearn.base']
    for mod_name in MOCK_MODULES:
        sys.modules[mod_name] = Mock()

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.napoleon',
    'sphinx.ext.mathjax'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'hsmmlearn'
copyright = u'2016, Joris Vankerschaver'
author = u'Joris Vankerschaver'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1.0'
# The full version, including alpha/beta/rc tags.
release = '0.1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# on_rtd is whether we are on readthedocs.org, this line of code grabbed from
# docs.readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# otherwise, readthedocs.org uses their theme by default, so no need to specify
# it

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'hsmmlearndoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'hsmmlearn.tex', u'hsmmlearn Documentation',
   u'Joris Vankerschaver', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'hsmmlearn', u'hsmmlearn Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'hsmmlearn', u'hsmmlearn Documentation',
   author, 'hsmmlearn', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False


#  Copyright 2015-present Scikit Flow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from sklearn import datasets, cross_validation, metrics
from sklearn import preprocessing

from tensorflow.contrib import skflow

# Load dataset
boston = datasets.load_boston()
X, y = boston.data, boston.target

# Split dataset into train / test
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y,
    test_size=0.2, random_state=42)

# scale data (training set) to 0 mean and unit Std. dev
scaler = preprocessing.StandardScaler()
X_train = scaler.fit_transform(X_train)

# Build 2 layer fully connected DNN with 10, 10 units respecitvely.
regressor = skflow.TensorFlowDNNRegressor(hidden_units=[10, 10],
    steps=5000, learning_rate=0.1, batch_size=1)

# Fit
regressor.fit(X_train, y_train)

# Predict and score
score = metrics.mean_squared_error(regressor.predict(scaler.fit_transform(X_test)), y_test)

print('MSE: {0:f}'.format(score))


#https://github.com/tflearn/tflearn/issues/180
from __future__ import division, print_function, absolute_import

import tflearn
from tflearn.data_utils import shuffle, to_categorical
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization, batch_normalization
from tflearn.layers.estimator import regression
from tflearn.data_utils import image_preloader
import skimage
from skimage import data
from skimage import filters
import os
from skimage import io
import numpy as np
from scipy import ndimage
import matplotlib.pyplot as plt

reds="../images/pictures/red/"
greens="../images/pictures/green/"
redshist="../images/histograms/red/"
greenshist="../images/histograms/green/"
directory=reds
histdirectory=redshist
for filename in os.listdir(directory):
    if filename.endswith(".jpg"):
	img = io.imread(os.path.join(directory, filename))
	hist, bin_edges = np.histogram(img, bins=255)
	bin_centers = 0.5*(bin_edges[:-1] + bin_edges[1:])
	binary_img = img > 0.8
	plt.figure(figsize=(1,1))
	fig, ax = plt.subplots(nrows=1, ncols=1) #http://stackoverflow.com/questions/9622163/save-plot-to-image-file-instead-of-displaying-it-using-matplotlib-so-it-can-be
	plt.plot(bin_centers, hist, lw=2)
	fig.savefig(os.path.join(histdirectory, filename), bbox_inches='tight')
	plt.close()
    else:
        continue


from __future__ import division, unicode_literals, absolute_import
import os, tempfile, copy, math, itertools, sys
import numpy as np
from operator import itemgetter
from itertools import product

try:
    import scipy

except:
    print('functions.py: no scipy, smoother() will not work()')

from siman import header
from siman.header import print_and_log, printlog, runBash, eV_A_to_J_m
from siman.small_functions import is_list_like, is_string_like, gunzip_file, makedir, grep_file, setting_sshpass

def unique_elements(seq, idfun=None):
   # return only unique_elements order preserving
   if idfun is None:
       def idfun(x): return x
   seen = {}
   result = []
   for item in seq:
       marker = idfun(item)
       # in old Python versions:
       # if seen.has_key(marker)
       # but in new ones:
       if marker in seen: continue
       seen[marker] = 1
       result.append(item)
   return result

def smoother(x, n, mul = 1, align = 1):
    """
    mul - additionally multiplies values
    #align - find first non-zero point and return it to zero
    #n - smooth value,
        if algo = 'gaus' than it is sigma
        use something like 0.8
        if algo = 'my'
            n of 10-15 is good
    """
    algo = 'gaus'
    # algo = 'my'

    if algo == 'my':
        x_smooth = []
        L = len(x)
        store = np.zeros((n,1),float)
        for u in range(L-n):
          for v in range(n):
               store[v] = x[u+v]
          av = float(sum(store)) / n
          x_smooth.append(av*mul)

        for u in range(L-n,L):
          for v in range(L-u-1):
               store[v] = x[u+v]
          av = float(sum(store)) / n
          x_smooth.append(av*mul)

    elif algo == 'gaus':
        x_smooth =x
        # x_smooth = scipy.ndimage.filters.median_filter(x,size =4)
        # print('sigma is ', n)
        x_smooth = scipy.ndimage.filters.gaussian_filter1d(x_smooth, n, order =0)
        # x_smooth = scipy.ndimage.interpolation.spline_filter1d(x, 4)

    else:
        x_smooth = x

    if align:
        # print(x_smooth[0])
        x_smooth[0] = 0
        # sys.exit()

    return np.asarray(x_smooth)

def run_on_server(command, addr = None):
    printlog('Running', command, 'on server ...')
    command = command.replace('\\', '/') # make sure is POSIX
    # sys.exit()

    # print(header.sshpass)
    # sys.exit()

    if addr is None:
        addr = header.cluster_address

    if header.ssh_object:
        # printlog('Using paramiko ...', imp = 'y')
        # if 'ne' in header.warnings:
        # sys.exit()

        out = header.ssh_object.run(command, noerror = True, printout = 'ne' in header.warnings)

    elif header.sshpass and header.sshpass == 'proxy':
        com = 'ssh -tt sdv sshpass -f '+ header.path2pass +' ssh '+addr+' "'+command+'"'
        # print(com)
        # sys.exit()

        out = runBash(com)
        # print(out)
        out = out.split('Connection to')[0] # remove last message Connection to ipaddress closed
        # sys.exit()

    elif header.sshpass:
        com = 'sshpass -f '+header.path2pass+' ssh '+addr+' "'+command+'"'
        # print(com)
        # sys.exit()

        out = runBash(com)

        # sys.exit()

    else:
        bash_comm = 'ssh '+addr+' "'+command+'"'
        # print(bash_comm)
        # sys.exit()
        out = runBash(bash_comm)

    out = out.split('#')[-1].strip()

    printlog(out)
    # print(out)
    # sys.exit()

    return out

def push_to_server(files = None, to = None,  addr = None):
    """
    if header.ssh_object then use paramiko
    to (str)     - path to remote folder !
    """
    if not is_list_like(files):
        files = [files]

    to = to.replace('\\', '/') # make sure is POSIX

    files_str = ' '.join(np.array(files ))

    command = ' mkdir -p {:}'.format( to )
    # print('asfsadfdsf', to)
    printlog('push_to_server():', command, run_on_server(command, addr))
    # sys.exit()

    printlog('push_to_server(): uploading files ', files, 'to', addr, to)

    if header.ssh_object:
        for file in files:
            # print(file, to)
            header.ssh_object.put(file,  to+'/'+os.path.basename(file) )
        out = ''

    elif header.sshpass and header.sshpass == 'proxy':
        com = 'tar cf - '+ files_str + ' | ssh sdv "sshpass -f ~/.ssh/p ssh '+addr+' \\"cd '+header.cluster_home+' && tar xvf -\\"" '
        # print(com)
        # sys.exit()
        out = runBash(com)

        # print(out)
        # sys.exit()
    elif header.sshpass:
        # if '@' not in addr:
        #     printlog('Error! Please provide address in the form user@address')
        # l = addr.split('@')
        # print(l)
        # user = l[0]
        # ad   = l[1]
        # com = 'rsync --rsh='+"'sshpass -f /home/aksenov/.ssh/p ssh' "  +' -uaz  '+files_str+ ' '+addr+':'+to
        com = 'rsync --rsh='+"'sshpass -f "+header.path2pass+" ssh' "  +' -uaz  '+files_str+ ' '+addr+':'+to

        # print(com)
        # sys.exit()
        out = runBash(com)

    else:
        out = runBash('rsync -uaz  '+files_str+ ' '+addr+':'+to)

    printlog(out)

    return out

def file_exists_on_server(file, addr):

    file = file.replace('\\', '/') # make sure is POSIX

    printlog('Checking existence of file', file, 'on server', addr )

    exist = run_on_server(' ls '+file, addr)

    # if header.ssh_object:
    #     exist = header.ssh_object.fexists(file)
    # else:
    #     exist = runBash('ssh '+addr+' ls '+file)

    if 'No such file' in exist:
        exist = ''
    else:
        exist = 'file exists'

    if exist:
        res = True
    else:
        res = False

    printlog('File exist? ', res)

    return res

def get_from_server(files = None, to = None, to_file = None,  addr = None, trygz = True):
    """
    Download files using either  paramiko (higher priority) or rsync;
    For paramiko header.ssh_object should be defined

    files (list of str)  - files on cluster to download
    to (str)      - path to local folder !
    to_file (str) - path to local file (if name should be changed); in this case len(files) should be 1

    The gz file is also checked

    RETURN
        result of download

    TODO:
    now for each file new connection is opened,
    copy them in one connection

    """
    # print(addr)
    # sys.exit()

    def download(file, to_file):

        # print(header.sshpass)
        if header.ssh_object:

            exist = file_exists_on_server(file, addr)
            # try:
            if exist:
                printlog('Using paramiko: ssh_object.get(): from  to ', file, to_file)
                header.ssh_object.get(file,  to_file  )
                out = ''
            # except FileNotFoundError:
            else:
                out = 'error, file not found'

        elif header.sshpass and header.sshpass == 'proxy':
            # com = 'ssh sdv "sshpass -f ~/.ssh/p ssh ' + addr + ' \\"tar zcf - '+ file +'\\"" | tar zxf - '+to_file # does not work?
            com = 'ssh sdv "sshpass -f ~/.ssh/p ssh ' + addr + ' \\"tar cf - '+ file +'\\"" > '+to_file
            # print('sshpass',com)
            # sys.exit()
            out = runBash(com)

        elif header.sshpass:
            #com = 'rsync --rsh='+"'sshpass -f /home/aksenov/.ssh/p ssh' "  +' -uaz  '+addr+':'+file+ ' '+to_file
            com = 'rsync --rsh='+"'sshpass -f "+header.path2pass+" ssh' "  +' -uaz  '+addr+':'+file+ ' '+to_file

            out = runBash(com)
            # print(addr)
            # sys.exit()

        else:
            # print(addr,file,to_file)
            out = runBash('rsync -uaz  '+addr+':'+file+ ' '+to_file)

        if 'error' in out:
            res = out
        else:
            res = 'OK'
            out = ''

        printlog('Download result is ', res)

        return out

    if '*' in files:
        printlog('get_from_server(): get by template')
        files = run_on_server('ls '+files, addr).splitlines()
        # print(files)
        # sys.exit()
        printlog('get_from_server(): I download', files)

    elif not is_list_like(files):
        files = [files]

    files = [file.replace('\\', '/') for file in files] #make sure the path is POSIX

    files_str = ', '.join(np.array(files ))
    printlog('Trying to download', files_str, 'from server', imp = 'n')

    for file in files:

        if not to and not to_file: #use temporary file
            with tempfile.NamedTemporaryFile() as f:
                to_file_l = f.name #system independent filename

        elif not to_file: #obtain filename
            to_file_l = os.path.join(to, os.path.basename(file) )

        else:
            to_file_l = to_file

        makedir(to_file_l)

        out = download(file, to_file_l)

        if out and trygz:

            printlog('File', file, 'does not exist, trying gz', imp = 'n')
            # run_on_server
            files = run_on_server(' ls '+file+'*', addr)
            file = files.split()[-1]
            # print(file)
            nz = file.count('gz')
            ext = '.gz'*nz

            # file+='.gz'
            to_file_l+=ext

            if file:
                out = download(file, to_file_l)
                printlog('    gz found with multiplicity', ext, imp = 'n')

                for i in range(nz):
                    printlog('unzipping', to_file_l)
                    gunzip_file(to_file_l)
                    to_file_l = to_file_l[:-3]
            else:
                printlog('    No gz either!', imp = 'n')

            # if '5247' in file:
            #     sys.exit()

    return out

def salary_inflation():
    """Calculate salary growth in Russia taking into account inflation"""
    inflation2000_2014 = [
     5.34,
     6.45,
     6.58,
     6.10,
     8.78,
     8.80,
     13.28,
     11.87,
     9.00 ,
     10.91,
     11.74,
     11.99,
     15.06,
     18.8,
     20.1]
    init_salary = 1500 # in jan 2000; other sources 2000 - very important
    for i,  l in enumerate( reversed(inflation2000_2014)  ):
        init_salary = (1+l/100)*init_salary
        print( init_salary, i+2000)

    salary2014 = 30000
    increase = salary2014/init_salary
    print( increase)

# salary_inflation()

def element_name_inv(el):
    el_dict = header.el_dict
    nu_dict = header.nu_dict
    # print type(el), el, type(str('sdf') )
    if is_string_like(el):
        try:
            elinv = el_dict[el]
        except:
            print_and_log("Error! Unknown element: " +str(el))
            raise RuntimeError
    else:
        el = int(el)
        try:
            elinv = nu_dict[el]
        except:
            print_and_log("Error! Unknown element: "+str(el))
            raise RuntimeError

    return elinv # inversed notion of element

invert = element_name_inv

def return_atoms_to_cell(st):

    st = st.return_atoms_to_cell()
    return st

def calc_ac(a1, c1, a2, c2, a_b = 0.1, c_b = 0.1, type = "two_atoms"):
    """
    Calculate values of hexagonal lattice parameters for cell with two different atoms.
    The used assumption is:
    1. Provided lattice constants are for large enougth cells, in which excess volume (dV) of impurity does not depend on the size of cell.
    2. Two atoms do not interact with each other, which allows to use dV(CO) = dV(C) + dV(O)

    Two regimes:
    two_atoms - calculate cell sizes if additional atom was added
    double_cell - if cell was doubled; only first cell and second_cell are needed

    Input:
    a1, c1 - lattice constants of cell with first impurity atom (first cell)
    a2, c2 - lattice constants of cell with second impurity atom (second cell)
    a_b, c_b - lattice constants of cell with pure hexagonal metall

    Output:
    a, c - lattice constants of cell with two atoms
    """
    hstring = ("%s    #on %s"% (traceback.extract_stack(None, 2)[0][3],   datetime.date.today() ) )
    if hstring != header.history[-1]: header.history.append( hstring  )

    A = (a1**2 * c1) + (a2**2 * c2) - (a_b**2 * c_b)
    B = 0.5 * (c1/a1 + c2/a2)
    C = ( (a1**2 * c1) + (a2**2 * c2) ) * 0.5 #sum of cell volumes divided by 2 since during the construction of new cell we will use multiplication by 2
    # print "A,B=",A,B
    a = (A/B)**(1./3)
    c = a * B
    a = round(a,5)
    c = round(c,5)
    print_and_log( "a, c, c/a for cell with pure    hcp ", a_b, c_b, round(c_b/a_b,4), imp ='y' )
    print_and_log( "a, c, c/a for cell with first  atom ", a1, c1, round(c1/a1,4), imp ='y' )
    print_and_log( "a, c, c/a for cell with second atom ", a2, c2, round(c2/a2,4), imp ='y' )

    #for double cell
    a3 = (C/B)**(1./3)
    c3 = a3 * B
    a3 = round(a3,5)
    c3 = round(c3,5)

    if type == "two_atoms":
        print_and_log( "a, c, c/a for cell with two   atoms ", a,  c, round(c/a,4), "# the same cell but with two atoms\n", imp ='y')
    elif type == "double_cell":
        print_and_log( "a, c, c/a for new cell              ", a3,  c3, round(c3/a3,4), "# for cell with V = V(first_cell) + V(second cell), but only for the case if V(second cell) == V(first_cell)", imp ='y')

    return a, c

def read_charge_den_vasp():
    """
    Read CHG vasp file and return ChargeDen object
    """
    class ChargeDen():
        """docstring for ChargeDen"""
        def __init__(self, ):
            # self.arg = arg

            pass

def rotation_matrix(axis,theta):
    axis = axis/math.sqrt(np.dot(axis,axis))
    a = math.cos(theta/2)
    b,c,d = -axis*math.sin(theta/2)
    return np.array([[a*a+b*b-c*c-d*d, 2*(b*c-a*d), 2*(b*d+a*c)],
                     [2*(b*c+a*d), a*a+c*c-b*b-d*d, 2*(c*d-a*b)],
                     [2*(b*d-a*c), 2*(c*d+a*b), a*a+d*d-b*b-c*c]])

def rotate():
    v = np.array([3,5,0])
    axis = np.array([4,4,1])
    theta = 1.2

    print(np.dot(rotation_matrix(axis,theta),v))
    # [ 2.74911638  4.77180932  1.91629719]

def rotation_matrix_from_vectors(vec1, vec2):
    """ Find the rotation matrix that aligns vec1 to vec2
    :param vec1: A 3d "source" vector
    :param vec2: A 3d "destination" vector
    :return mat: A transform matrix (3x3) which when applied to vec1, aligns it with vec2.
    """
    a, b = (vec1 / np.linalg.norm(vec1)).reshape(3), (vec2 / np.linalg.norm(vec2)).reshape(3)
    v = np.cross(a, b)
    c = np.dot(a, b)
    s = np.linalg.norm(v)
    kmat = np.array([[0, -v[2], v[1]], [v[2], 0, -v[0]], [-v[1], v[0], 0]])
    rotation_matrix = np.eye(3) + kmat + kmat.dot(kmat) * ((1 - c) / (s ** 2))
    return rotation_matrix

def plot_charge_den():
    """Test function; Was not used"""
    from mpl_toolkits.mplot3d import axes3d
    import matplotlib.pyplot as plt
    from matplotlib import cm

    fig = plt.figure()
    ax = fig.gca(projection='3d')
    X, Y, Z = axes3d.get_test_data(0.05)
    # print X
    # print Y
    # print Z

    ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3)
    # cset = ax.contourf(X, Y, Z, zdir='z', offset=-100, cmap=cm.coolwarm)
    # cset = ax.contourf(X, Y, Z, zdir='x', offset=-40, cmap=cm.coolwarm)
    # cset = ax.contourf(X, Y, Z, zdir='y', offset=40, cmap=cm.coolwarm)

    ax.set_xlabel('X')
    ax.set_xlim(-40, 40)
    ax.set_ylabel('Y')
    ax.set_ylim(-40, 40)
    ax.set_zlabel('Z')
    ax.set_zlim(-100, 100)

    plt.show()

    return

def plot_interaction(calclist, calc):
    """
    For calculation of interaction parameter alpha;
    Take in mind that this parameter is obtained under aproximation of redular solution
    """
    e_seg = []
    dX = []
    for id in calclist:
        Xgb = calc[id].Xgb
        X = calc[id].X
        dX.append(Xgb/1 - X)
        e_seg.append(calc[id].e_seg)
        # print calc[id].e_seg
        # print calc[id].X
        #print dX
    coeffs1 = np.polyfit(dX, e_seg, 1)

    fit_func1 = np.poly1d(coeffs1)
    print( "list of seg energies: ", e_seg  )
    print( "list of dX          : ", dX  )

    print( "Fitting using linear function:"  )
    print( fit_func1  )
    print( "E_seg0 = {0:0.0f} meV, standart enthalpy of segregation".format(fit_func1[0])  )
    print( "alpha  = {0:0.0f} meV, interaction coefficient".format(-fit_func1[1]/2)  )

    return

def calculate_voronoi(self, state = 'end'):
    # By default two quantities per atom are calculated by this compute.
    # The first is the volume of the Voronoi cell around each atom.
    # Any point in an atom's Voronoi cell is closer to that atom than any other.
    # The second is the number of faces of the Voronoi cell, which
    # is also the number of nearest neighbors of the atom in the middle of the cell.
    # state - init or end; if init then saved in self.init.vorovol; if end than saved in self.vorovol

    write_lammps(self, state, filepath = 'voronoi_analysis/structure.lammps') #write structure for lammps
    runBash("rm voronoi_analysis/dump.voro; /home/aksenov/installed/lammps-1Feb14/src/lmp_serial < voronoi_analysis/voronoi.in > voronoi_analysis/log")

    if state == 'end':
        self.vorovol = []
        self.vorofaces = []
        vorovol = self.vorovol
        vorofaces = self.vorofaces
    elif state == 'init':
        self.init.vorovol = []
        self.init.vorofaces = []
        vorovol = self.init.vorovol
        vorofaces = self.init.vorofaces

    vsum=0
    wlist = []
    with open('voronoi_analysis/dump.voro','r') as volfile:  #analyze dump.voro
        for line in volfile:
            if 'ITEM: ATOMS ' in line:
                break
        for line in volfile:
            ll = line.split()
            if int(ll[1]) > 1:
                wlist.append( [ll[0], ll[5], ll[6], ll[2]] )
            # print 'Volume of atom ',ll[0],'is', ll[5]
            vsum= vsum+float(ll[5])
        print_and_log( 'Check total volume ', vsum, self.end.vol)

        wlist.sort(key = itemgetter(0)) #sort according to the position of atoms
        print_and_log( "atom #, voronoi vol, voronoi faces, x coordinate: ", )
        print_and_log( wlist)
        for w in wlist:
            vorovol.append(float(w[1]))
            vorofaces.append(int(w[2]))
        # print 'Voro vol  ',self.end.vorovol
        # print 'Voro faces',self.end.vorofaces
        # print len(wlist)
    if hasattr(self, 'vorovol'):
        voro = ''
        if len(vorovol) == 2: #C and O
            voro = " {0:5.2f} & {1:2d} & {2:5.2f} & {3:2d} ".format(vorovol[0], vorofaces[0], vorovol[1], vorofaces[1]  ).center(25)
        else:
            voro = " {0:5.2f} & {1:2d} ".format(vorovol[0], vorofaces[0] ).center(25)
        voro+='&'
    else:
        voro = ""
    print_and_log( "Voronoi volume = ", voro, imp = 'y')
    return voro

def log_history(hstring):
    try:
        if hstring != header.history[-1]: header.history.append( hstring  )
    except:
        header.history.append( hstring  )
    return

def gb_energy_volume(gb,bulk):
    if (gb.end.rprimd[1] != bulk.end.rprimd[1]).any() or (gb.end.rprimd[2] != bulk.end.rprimd[2]).any():
        print_and_log("Warning! You are trying to calculate gb_energy from cells with different lateral sizes:"+str(gb.end.rprimd)+" "+str(bulk.end.rprimd)+"\n")
    #print bulk.vol
    V_1at = bulk.vol / bulk.natom #* to_ang**3

    E_1at = bulk.energy_sigma0 / bulk.natom
    A = np.linalg.norm( np.cross(gb.end.rprimd[1], gb.end.rprimd[2])  ) #surface area of gb
    #print A
    gb.v_gb =      ( gb.vol              - V_1at * gb.natom) / A / 2. * 1000
    gb.e_gb =      ( gb.energy_sigma0    - E_1at * gb.natom) / A / 2. * eV_A_to_J_m * 1000
    gb.e_gb_init = ( gb.list_e_sigma0[0] - E_1at * gb.natom) / A / 2. * eV_A_to_J_m * 1000
    gb.bulk_extpress = bulk.extpress
    #print "Calc %s; e_gb_init = %.3f J/m^2; e_gb = %.3f J/m; v_gb = %.3f angstrom "%(gb.name, gb.e_gb_init, gb.e_gb, gb.v_gb )
    outst = "%15s&%7.0f&%7.0f"%(gb.name, gb.e_gb, gb.v_gb)
    return outst

def headers():
    j = (7,12,14,7,8,9,9,5,5,20,5,20,8,12,20,8,5,8,8)
    d="&"
    header_for_bands= "Set".ljust(j[0])+d+"Etot".center(j[1])+d+"a1,a2".center(j[2])+d+"c".center(j[3])\
                +d+"time, m".center(j[4])+d+"ittime, s".center(j[5])+d+"Nmd,Avr.".rjust(j[6])+d\
                +"Warn!"+d+"nband"+d+"Added, \%"+"\\\\"

    header_for_ecut= "Set".ljust(j[0])+d+"Etot".center(j[1])+d+"a1,a2".center(j[2])+d+"c".center(j[3])\
                +d+"time, m".center(j[4])+d+"ittime, s".center(j[5])+d+"Nmd,Avr.".rjust(j[6])+d\
                +"Warn!"+d+"Ecut,eV"+"\\\\"

    header_for_npar= "Set".ljust(j[0])+d+"Etot".center(j[1])+d+"a1,a2".center(j[2])+d+"c".center(j[3])\
                +d+"time, m".center(j[4])+d+"ittime, s".center(j[5])+d+"Nmd,Avr.".rjust(j[6])+d\
                +"Warn!"+d+"NPAR".center(j[16])+d+"LPLANE".center(j[17])+"\\\\"

    header_for_kpoints= "Set".ljust(j[0])+d+"Etot".center(j[1])+d+"a1,a2".center(j[2])+d+"c".center(j[3])\
                +d+"time, m".center(j[4])+d+"ittime, s".center(j[5])+d+"Nmd,Avr.".rjust(j[6])+d\
                +"Warn!"+d+"k-mesh".center(j[8])+d+"k-spacings".center(j[9])+d+"nkpt".center(j[10])+"\\\\"
    header_for_tsmear= "Set".ljust(j[0])+d+"Etot".center(j[1])+d+"a1,a2".center(j[2])+d+"c".center(j[3])\
                +d+"time, m".center(j[4])+d+"ittime, s".center(j[5])+d+"Nmd,Avr.".rjust(j[6])+d\
                +"Warn!"+d+"k-mesh".center(j[8])+d+"tsmear, meV".center(j[13])+d+"Smearing error, meV/atom".center(j[14])+"\\\\"

    header_for_stress= "Set".ljust(j[0])+d+"Etot".center(j[1])+d+"a1,a2".center(j[2])+d+"c".center(j[3])\
                +d+"time, m".center(j[4])+d+"ittime, s".center(j[5])+d+"Nmd,Avr.".rjust(j[6])+d\
                +"Warn!"+d+"Stress, intr u.*1000".center(j[11])+d+"Pressure, MPa".center(j[12])
    #print "\\hline"
    return header_for_kpoints

def read_vectors(token, number_of_vectors, list_of_words, type_func = None, lists = False):
    """Returns the list of numpy vectors for the last match"""
    # lists - return list of lists instead list of vectors

    if type_func is None:
        type_func = lambda a : float(a)

    number_of_matches = list_of_words.count( token )
    if number_of_matches == 0:
        #print_and_log("Warning token '"+token+"' was not found! return empty\n")
        return [None]

    if number_of_matches > 1:
        print_and_log("Warning token '"+token+"' was found more than one times\n")
        raise RuntimeError

    index = list_of_words.index(token, number_of_matches - 1 )     #Return the index of the last match
    #print list_of_words[index]
    list_of_vectors = []
    list_of_lists = []
    vector = np.zeros((3))
    for i in range(number_of_vectors):
        vector[0] = type_func(list_of_words[index + 1])
        vector[1] = type_func(list_of_words[index + 2])
        vector[2] = type_func(list_of_words[index + 3])
        list3 = []
        for j in 1,2,3:
            list3.append(type_func(list_of_words[index + j]) )

        index+=3
        list_of_vectors.append(vector.copy())
        list_of_lists.append(list3)

    if lists:
        out = list_of_lists
    else:
        out = list_of_vectors

    return out

def read_string(token, length, string):
    sh = len(token)+1
    i = string.find(token)+sh
    # print('length', i, i+length)
    # sys.exit()
    if i is -1:
        return ''
    else:
        return string[i:i+length]

def read_list(token, number_of_elements, ttype, list_of_words):
    """Input is token to find, number of elements to read, type of elements and list of words,
    where to search
    Returns the list of elements for the last match"""

    number_of_matches = list_of_words.count( token )

    #if number_of_elements == 0:        raise RuntimeError

    if number_of_matches > 1:
        print_and_log("Warning token '"+token+"' was found more than one times\n")
        raise RuntimeError

    if number_of_matches == 0 or number_of_elements == 0:
        #print_and_log("Warning token '"+token+"' was not found or asked number of elements is zero! set to [None]\n")
        #if ttype == str:
        #    return ['']*number_of_elements
        #else:
        #    return [0]*number_of_elements
        return [None]

    try:
        index = list_of_words.index(token, number_of_matches - 1 )     #Return the index of the last match

    except ValueError:
        print_and_log("Warning!, token "+token+" was not found. I return [None]!\n")
        return [None]

    index+=1 #the position of token value
    list_of_elements = []

    #define function dependig on type:

    if   ttype == int  :
        def convert(a):
            return int(a)

    elif ttype == float:
        def convert(a):
            # print a
            return float(a)

    elif ttype == str  :
        def convert(a):
            return str(a)

    #print list_of_words[index], type(list_of_words[index])
    if list_of_words[index] == "None"  :
        def convert(a):
            return [None]

    #Make convertion
    for i in range(number_of_elements):

        if 'None' in list_of_words[index]:
            list_of_elements.append(None)
        else:
            list_of_elements.append(    convert(  list_of_words[index]   )     )

        index+=1

    return list_of_elements

def words(fileobj):
    """Generator of words. However does not allow to use methods of list for returned"""
    for line in fileobj:
        for word in line.split():
            yield word

def server_cp(copy_file, to, gz = True, scratch = False, new_filename = None):

    if scratch:
        if not header.PATH2ARCHIVE:
            printlog('Warning! PATH2ARCHIVE is empty! Please put path archive in ~/simanrc.py or ./project_conf.py ')

        copy_file = header.PATH2ARCHIVE + '/' + copy_file
    else:
        copy_file = header.project_path_cluster + '/' + copy_file

    filename = os.path.basename(copy_file)

    if new_filename is None:
        new_filename = filename

    if gz:
        command = 'cp '+copy_file + ' ' + to +'/'+new_filename + '.gz ; gunzip -f '+ to+ '/'+new_filename+'.gz'
    else:
        command = 'cp '+copy_file + ' ' + to +'/'+new_filename

    printlog('Running on server', command, imp = '')
    if file_exists_on_server(copy_file, header.cluster_address):
        out = run_on_server(command, addr = header.cluster_address)
        printlog('Output of run_on_server', out, imp = '')
    else:
        out = 'error, file does not exist on server: '+copy_file
    return out

def wrapper_cp_on_server(file, to, new_filename = None):
    """
    tries iterativly scratch and gz
    """
    copy_to   = to

    copy_file = file

    filename = os.path.basename(file)
    if new_filename:
        app = 'with new name '+new_filename
    else:
        app = ''

    for s, gz in product([0,1], ['', '.gz']):

        printlog('scratch, gz:', s, gz)

        out = server_cp(copy_file+gz, to = to, gz = gz, scratch = s, new_filename = new_filename)

        if out == '':
            printlog('File', filename, 'was succesfully copied to',to, app, imp = 'y')
            break
        # else:
    else:
        printlog('Warning! File was not copied, probably it does not exist. Try using header.warnings = "neyY" for more details', imp = 'y')

    return

def update_incar(parameter = None, value = None, u_ramp_step = None, write = True, f = None, run = False, st = None):
    """Modifications of INCAR. Take attention that *parameter* will be changed to new *value*
    if it only already exist in INCAR.  *u_ramp_step*-current step to determine u,
    *write*-sometimes just the return value is needed.
    Returns U value corresponding to *u_ramp_step*.
    """

    self = st
    u_step = None
    if parameter == 'LDAUU':
        #Update only non-zero elements of LDAUU with value

        set_LDAUU_list = self.set.vasp_params['LDAUU']
        new_LDAUU_list = copy.deepcopy(set_LDAUU_list)

        # print set_LDAUU_list
        u_step = 0.0
        for i, u in enumerate(set_LDAUU_list):
            if u == 0:
                continue
            u_step = np.linspace(0, u, self.set.u_ramping_nstep)[u_ramp_step]
            u_step = np.round(u_step, 1)
            # new_LDAUU_list[i] = value
            new_LDAUU_list[i] = u_step

        new_LDAUU = 'LDAUU = '+' '.join(['{:}']*len(new_LDAUU_list)).format(*new_LDAUU_list)

        command = "sed -i.bak '/LDAUU/c\\" + new_LDAUU + "' INCAR\n"
        #print('u_step',u_step)
        #sys.exit()

    elif parameter == 'MAGMOM':

        new_incar_string = parameter + ' = ' + ' '.join(['{:}']*len(value)).format(*value)
        command = "sed -i.bak '/"+parameter+"/c\\" + new_incar_string + "' INCAR\n"

    # elif parameter in ['IMAGES', 'ISPIN']:
    else:

        new_incar_string = parameter + ' = ' + str(value)
        command = "sed -i.bak '/"+parameter+"/c\\" + new_incar_string + "' INCAR\n"

    if write and f:
        f.write(command)

    if run:
        runBash(command)

    return  u_step #for last element

def check_output(filename, check_string, load):
    """
    Check if file exist and it is finished by search for check_string
    """

    if filename and os.path.exists(filename):

        out = grep_file(check_string, filename, reverse = True)

        printlog('The grep result of',filename, 'is:', out)
        # sys.exit()
        if check_string in out or 'un' in load:
            state = '4. Finished'
        else:
            state = '5. Broken outcar'

    else:
        state = '5. no OUTCAR'

    return state


import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

def hist_dist(title, distribution_tensor, hist_range=(-4, 4)):
    """
    Display histogram of a TF distribution
    """
    with tf.Session() as sess:
        values = sess.run(distribution_tensor)

    plt.title(title)
    plt.hist(values, np.linspace(*hist_range, num=len(values)/2))
    plt.show()

def _get_loss_acc(dataset, weights):
    """
    Get losses and validation accuracy of example neural network
    """
    batch_size = 128
    epochs = 2
    learning_rate = 0.001

    features = tf.placeholder(tf.float32)
    labels = tf.placeholder(tf.float32)
    learn_rate = tf.placeholder(tf.float32)

    biases = [
        tf.Variable(tf.zeros([256])),
        tf.Variable(tf.zeros([128])),
        tf.Variable(tf.zeros([dataset.train.labels.shape[1]]))
    ]

    # Layers
    layer_1 = tf.nn.relu(tf.matmul(features, weights[0]) + biases[0])
    layer_2 = tf.nn.relu(tf.matmul(layer_1, weights[1]) + biases[1])
    logits = tf.matmul(layer_2, weights[2]) + biases[2]

    # Training loss
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels))

    # Optimizer
    optimizer = tf.train.AdamOptimizer(learn_rate).minimize(loss)

    # Accuracy
    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    # Measurements use for graphing loss
    loss_batch = []

    with tf.Session() as session:
        session.run(tf.global_variables_initializer())
        batch_count = int((dataset.train.num_examples / batch_size))

        # The training cycle
        for epoch_i in range(epochs):
            for batch_i in range(batch_count):
                batch_features, batch_labels = dataset.train.next_batch(batch_size)

                # Run optimizer and get loss
                session.run(
                    optimizer,
                    feed_dict={features: batch_features, labels: batch_labels, learn_rate: learning_rate})
                l = session.run(
                    loss,
                    feed_dict={features: batch_features, labels: batch_labels, learn_rate: learning_rate})
                loss_batch.append(l)

        valid_acc = session.run(
            accuracy,
            feed_dict={features: dataset.validation.images, labels: dataset.validation.labels, learn_rate: 1.0})

    # Hack to Reset batches
    dataset.train._index_in_epoch = 0
    dataset.train._epochs_completed = 0

    return loss_batch, valid_acc

def compare_init_weights(
        dataset,
        title,
        weight_init_list,
        plot_n_batches=100):
    """
    Plot loss and print stats of weights using an example neural network
    """
    colors = ['r', 'b', 'g', 'c', 'y', 'k']
    label_accs = []
    label_loss = []

    assert len(weight_init_list) <= len(colors), 'Too many inital weights to plot'

    for i, (weights, label) in enumerate(weight_init_list):
        loss, val_acc = _get_loss_acc(dataset, weights)

        plt.plot(loss[:plot_n_batches], colors[i], label=label)
        label_accs.append((label, val_acc))
        label_loss.append((label, loss[-1]))

    plt.title(title)
    plt.xlabel('Batches')
    plt.ylabel('Loss')
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.show()

    print('After 858 Batches (2 Epochs):')
    print('Validation Accuracy')
    for label, val_acc in label_accs:
        print('  {:7.3f}% -- {}'.format(val_acc*100, label))
    print('Loss')
    for label, loss in label_loss:
        print('  {:7.3f}  -- {}'.format(loss, label))