#!/usr/bin/env python
#
# Copyright 2013 Free Software Foundation, Inc.
#
# This file is part of GNU Radio
#
# GNU Radio is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GNU Radio is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNU Radio; see the file COPYING.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street,
# Boston, MA 02110-1301, USA.
#

from gnuradio import gr, filter
from gnuradio import analog
from gnuradio import blocks
from gnuradio import eng_notation
from gnuradio.eng_option import eng_option
from optparse import OptionParser
import sys

try:
    import scipy
except ImportError:
    print "Error: could not import scipy (http://www.scipy.org/)"
    sys.exit(1)

try:
    import pylab
except ImportError:
    print "Error: could not import pylab (http://matplotlib.sourceforge.net/)"
    sys.exit(1)

class example_fir_filter_ccc(gr.top_block):
    def __init__(self, N, fs, bw, tw, atten, D):
        gr.top_block.__init__(self)

        self._nsamps = N
        self._fs = fs
        self._bw = bw
        self._tw = tw
        self._at = atten
        self._decim = D
        taps = filter.firdes.low_pass_2(1, self._fs, self._bw, self._tw, self._at)
        print "Num. Taps: ", len(taps)

        self.src  = analog.noise_source_c(analog.GR_GAUSSIAN, 1)
        self.head = blocks.head(gr.sizeof_gr_complex, self._nsamps)

        self.filt0 = filter.fir_filter_ccc(self._decim, taps)

        self.vsnk_src = blocks.vector_sink_c()
        self.vsnk_out = blocks.vector_sink_c()

        self.connect(self.src, self.head, self.vsnk_src)
        self.connect(self.head, self.filt0, self.vsnk_out)

def main():
    parser = OptionParser(option_class=eng_option, conflict_handler="resolve")
    parser.add_option("-N", "--nsamples", type="int", default=10000,
                      help="Number of samples to process [default=%default]")
    parser.add_option("-s", "--samplerate", type="eng_float", default=8000,
                      help="System sample rate [default=%default]")
    parser.add_option("-B", "--bandwidth", type="eng_float", default=1000,
                      help="Filter bandwidth [default=%default]")
    parser.add_option("-T", "--transition", type="eng_float", default=100,
                      help="Transition band [default=%default]")
    parser.add_option("-A", "--attenuation", type="eng_float", default=80,
                      help="Stopband attenuation [default=%default]")
    parser.add_option("-D", "--decimation", type="int", default=1,
                      help="Decmation factor [default=%default]")
    (options, args) = parser.parse_args ()

    put = example_fir_filter_ccc(options.nsamples,
                                 options.samplerate,
                                 options.bandwidth,
                                 options.transition,
                                 options.attenuation,
                                 options.decimation)
    put.run()

    data_src = scipy.array(put.vsnk_src.data())
    data_snk = scipy.array(put.vsnk_out.data())

    # Plot the signals PSDs
    nfft = 1024
    f1 = pylab.figure(1, figsize=(12,10))
    s1 = f1.add_subplot(1,1,1)
    s1.psd(data_src, NFFT=nfft, noverlap=nfft/4,
           Fs=options.samplerate)
    s1.psd(data_snk, NFFT=nfft, noverlap=nfft/4,
           Fs=options.samplerate)

    f2 = pylab.figure(2, figsize=(12,10))
    s2 = f2.add_subplot(1,1,1)
    s2.plot(data_src)
    s2.plot(data_snk.real, 'g')

    pylab.show()

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        pass


"""
Helper functions for determining which tests to skip.
"""

__authors__ = "Ian Goodfellow"
__copyright__ = "Copyright 2010-2012, Universite de Montreal"
__credits__ = ["Ian Goodfellow"]
__license__ = "3-clause BSD"
__maintainer__ = "LISA Lab"
__email__ = "pylearn-dev@googlegroups"
from nose.plugins.skip import SkipTest
import os
from theano.sandbox import cuda

scipy_works = True
try:
    import scipy
except ImportError:
    # pyflakes gets mad if you set scipy to None here
    scipy_works = False

sklearn_works = True
try:
    import sklearn
except ImportError:
    sklearn_works = False

h5py_works = True
try:
    import h5py
except ImportError:
    h5py_works = False

matplotlib_works = True
try:
    from matplotlib import pyplot
except ImportError:
    matplotlib_works = False

def skip_if_no_data():
    if 'PYLEARN2_DATA_PATH' not in os.environ:
        raise SkipTest()

def skip_if_no_scipy():
    if not scipy_works:
        raise SkipTest()

def skip_if_no_sklearn():
    if not sklearn_works:
        raise SkipTest()

def skip_if_no_gpu():
    if cuda.cuda_available == False:
        raise SkipTest('Optional package cuda disabled.')

def skip_if_no_h5py():
    if not h5py_works:
        raise SkipTest()

def skip_if_no_matplotlib():
    if not matplotlib_works:
        raise SkipTest("matplotlib and pyplot are not available")


# -*- coding: utf-8 -*-
"""
Created on Tue Jul 22 07:54:05 2014

@author: charleslelosq
Carnegie Institution for Science

"""
import sys
sys.path.append("/Users/charleslelosq/Documents/RamPy/lib-charles/")

import csv
import numpy as np
import scipy
import matplotlib
import matplotlib.gridspec as gridspec
from pylab import *
from StringIO import StringIO
from scipy import interpolate

# to fit spectra we use the lmfit software of Matt Newville, CARS, university of Chicago, available on the web
from lmfit import minimize, Minimizer, Parameters, Parameter, report_fit, fit_report

from spectratools import *  #Charles' libraries and functions

from Tkinter import *
import tkMessageBox
from tkFileDialog import askopenfilename

#### We define a set of functions that will be used for fitting data
#### unfortunatly, as we use lmfit (which is convenient because it can fix or release
#### easily the parameters) we are not able to use arrays for parameters...
#### so it is a little bit long to write all the things, but in a way quite robust also...
#### gaussian and pseudovoigt functions are available in spectratools
#### if you need a voigt, fix the gaussian-to-lorentzian ratio to 1 in the parameter definition before
#### doing the data fit
def residual(pars, x, data=None, eps=None):
    # unpack parameters:
    #  extract .value attribute for each parameter
    a1 = pars['a1'].value
    a2 = pars['a2'].value

    f1 = pars['f1'].value
    f2 = pars['f2'].value

    l1 = pars['l1'].value
    l2 = pars['l2'].value

    # Gaussian model

    peak1 = gaussian(x,a1,f1,l1)
    peak2 = gaussian(x,a2,f2,l2)

    model = peak1 + peak2

    if data is None:
        return model, peak1, peak2
    if eps is None:
        return (model - data)
    return (model - data)/eps

##### CORE OF THE CALCULATION BELOW

#### CALLING THE DATA NAMES
tkMessageBox.showinfo(
            "Open file",
            "Please open the list of spectra")

Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
filename = askopenfilename() # show an "Open" dialog box and return the path to the selected file
with open(filename) as inputfile:
    results = list(csv.reader(inputfile)) # we read the data list

#### LOOP FOR BEING ABLE TO TREAT MULTIPLE DATA
#### WARNING: OUTPUT ARE AUTOMATICALLY GENERATED IN A DIRECTORY CALLED "DECONV"
#### (see end) THAT SHOULD BE PRESENT !!!!!!!!!!
for lg in range(len(results)):
    name = str(results[lg]).strip('[]')
    name = name[1:-1] # to remove unwanted ""
    sample = np.genfromtxt(name) # get the sample to deconvolute

    # we set here the lower and higher bonds for the interest region
    lb = 4700 ### MAY NEED TO AJUST THAT
    hb = 6000

    interestspectra = sample[np.where((sample[:,0] > lb)&(sample[:,0] < hb))]
    ese0 = interestspectra[:,2]/abs(interestspectra[:,1]) #take ese  as a percentage, we assume that the treatment was made correctly for error determination... if not, please put  sigma = None
    interestspectra[:,1] = interestspectra[:,1]/np.amax(interestspectra[:,1])*100 # normalise spectra to maximum, easier to handle after
    sigma = abs(ese0*interestspectra[:,1]) #calculate good ese
    #sigma = None # you can activate that if you are not sure about the errors

    xfit = interestspectra[:,0] # region to be fitted
    data = interestspectra[:,1] # region to be fitted

    params = Parameters()
    ####################### FOR MELT:
    ####################### COMMENT IF NOT WANTED
    #               (Name,  Value,  Vary,   Min,  Max,  Expr)
    params.add_many(('a1',   1,   True, 0,      None,  None),
                    ('f1',   5200,  True, 750,    None,  None),
                    ('l1',   1,  True, 0,      None,  None),
                    ('a2',   1,  True, 0,      None,  None),
                    ('f2',   5400,  True, None,   None,  None),
                    ('l2',   1,  True, None,   None,  None))

    result = minimize(residual_melt, params, args=(xfit, data)) # fit data with leastsq model from scipy
    model = fit_report(params) # the report
    yout, peak1,peak2,= residual_melt(params,xfit) # the different peaks

    #### We just calculate the different areas up to 4700 cmm-1 and those of the gaussians
    # Select interest areas for calculating the areas of OH and H2Omol peaks
    intarea45 = sample[np.where((sample[:,0]> 4100) & (sample[:,0]<4700))]
    area4500 = np.trapz(intarea45[:,1],intarea45[:,0])
    esearea4500 = 1/sqrt(area4500) # We assume that RELATIVE errors on areas are globally equal to 1/sqrt(Area)

    # now for the gaussians
    # unpack parameters:
    #  extract .value attribute for each parameter
    a1 = pars['a1'].value
    a2 = pars['a2'].value

    l1 = pars['l1'].value
    l2 = pars['l2'].value

    AireG1 = gaussianarea(a1,l1)
    AireG2 = gaussianarea(a2,l2)

    ##### WE DO A NICE FIGURE THAT CAN BE IMPROVED FOR PUBLICATION
    fig = figure()
    plot(sample[:,0],sample[:,1],'k-')
    plot(xfit,yout,'r-')
    plot(xfit,peak1,'b-')
    plot(xfit,peak2,'b-')

    xlim(lb,hb)
    ylim(0,np.max(sample[:,1]))
    xlabel("Wavenumber, cm$^{-1}$", fontsize = 18, fontweight = "bold")
    ylabel("Absorption, a. u.", fontsize = 18, fontweight = "bold")

    text(4000,np.max(intarea45[:,1])+0.03*np.max(intarea45[:,1]),('Area OH: \n'+'%.1f' % area4500),color='b',fontsize = 16)
    text(4650,a1 + 0.05*a1,('Area pic 1$: \n'+ '%.1f' % AireG1),color='b',fontsize = 16)
    text(5000,a2 + 0.05*a2,('OH/H$_2$O$_{mol}$: \n'+'%.3f' % ratioOH_H2O+'\n+/-'+'%.3f' % eseratioOH_H2O),color='r',fontsize = 16)

    ##### output of data, fitted peaks, parameters, and the figure in pdf
    ##### all goes into the ./deconv/ folder
    name.rfind('/')
    nameout = name[name.rfind('/')+1::]
    namesample = nameout[0:nameout.find('.')]
    pathint = str('/deconv/') # the output folder
    ext1 = '_ydec.txt'
    ext2 = '_params.txt'
    ext3 = '.pdf'
    pathout1 = pathbeg+pathint+namesample+ext1
    pathout2 = pathbeg+pathint+namesample+ext2
    pathout3 = pathbeg+pathint+namesample+ext3
    matout = np.vstack((xfit,data,yout,peak1,peak2))
    matout = np.transpose(matout)
    np.savetxt(pathout1,matout) # saving the arrays of spectra
    fd = os.open( pathout2, os.O_RDWR|os.O_CREAT ) # Open a file and create it if it do not exist
    fo = os.fdopen(fd, "w+") # Now get a file object for the above file.
    fo.write(model) # write the parameters in it
    fo.close()
    savefig(pathout3) # save the figure


"""
========================================
Plot multi-class SGD on the iris dataset
========================================

Plot decision surface of multi-class SGD on iris dataset.
The hyperplanes corresponding to the three one-versus-all (OVA) classifiers
are represented by the dashed lines.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.linear_model import SGDClassifier

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target
colors = "bry"

# shuffle
idx = np.arange(X.shape[0])
np.random.seed(13)
np.random.shuffle(idx)
X = X[idx]
y = y[idx]

# standardize
mean = X.mean(axis=0)
std = X.std(axis=0)
X = (X - mean) / std

h = .02  # step size in the mesh

clf = SGDClassifier(alpha=0.001, n_iter=100).fit(X, y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, m_max]x[y_min, y_max].
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
# Put the result into a color plot
Z = Z.reshape(xx.shape)
cs = plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
plt.axis('tight')

# Plot also the training points
for i, color in zip(clf.classes_, colors):
    idx = np.where(y == i)
    plt.scatter(X[idx, 0], X[idx, 1], c=color, label=iris.target_names[i],
                cmap=plt.cm.Paired)
plt.title("Decision surface of multi-class SGD")
plt.axis('tight')

# Plot the three one-against-all classifiers
xmin, xmax = plt.xlim()
ymin, ymax = plt.ylim()
coef = clf.coef_
intercept = clf.intercept_

def plot_hyperplane(c, color):
    def line(x0):
        return (-(x0 * coef[c, 0]) - intercept[c]) / coef[c, 1]

    plt.plot([xmin, xmax], [line(xmin), line(xmax)],
             ls="--", color=color)

for i, color in zip(clf.classes_, colors):
    plot_hyperplane(i, color)
plt.legend()
plt.show()


#!/usr/bin/env python
'''
Make a set of images with a single colormap, norm, and colorbar.

It also illustrates colorbar tick labelling with a multiplier.
'''

from matplotlib.pyplot import figure, show, axes, sci
from matplotlib import cm, colors
from matplotlib.font_manager import FontProperties
from numpy import amin, amax, ravel
from numpy.random import rand

Nr = 3
Nc = 2

fig = figure()
cmap = cm.cool

figtitle = 'Multiple images'
t = fig.text(0.5, 0.95, figtitle,
               horizontalalignment='center',
               fontproperties=FontProperties(size=16))

cax = fig.add_axes([0.2, 0.08, 0.6, 0.04])

w = 0.4
h = 0.22
ax = []
images = []
vmin = 1e40
vmax = -1e40
for i in range(Nr):
    for j in range(Nc):
        pos = [0.075 + j*1.1*w, 0.18 + i*1.2*h, w, h]
        a = fig.add_axes(pos)
        if i > 0:
            a.set_xticklabels([])
        # Make some fake data with a range that varies
        # somewhat from one plot to the next.
        data =((1+i+j)/10.0)*rand(10,20)*1e-6
        dd = ravel(data)
        # Manually find the min and max of all colors for
        # use in setting the color scale.
        vmin = min(vmin, amin(dd))
        vmax = max(vmax, amax(dd))
        images.append(a.imshow(data, cmap=cmap))

        ax.append(a)

# Set the first image as the master, with all the others
# observing it for changes in cmap or norm.

class ImageFollower:
    'update image in response to changes in clim or cmap on another image'
    def __init__(self, follower):
        self.follower = follower
    def __call__(self, leader):
        self.follower.set_cmap(leader.get_cmap())
        self.follower.set_clim(leader.get_clim())

norm = colors.Normalize(vmin=vmin, vmax=vmax)
for i, im in enumerate(images):
    im.set_norm(norm)
    if i > 0:
        images[0].callbacksSM.connect('changed', ImageFollower(im))

# The colorbar is also based on this master image.
fig.colorbar(images[0], cax, orientation='horizontal')

# We need the following only if we want to run this interactively and
# modify the colormap:

axes(ax[0])     # Return the current axes to the first one,
sci(images[0])  # because the current image must be in current axes.

show()


"""
========================
IncrementalPCA benchmark
========================

Benchmarks for IncrementalPCA

"""

import numpy as np
import gc
from time import time
from collections import defaultdict
import matplotlib.pyplot as plt
from sklearn.datasets import fetch_lfw_people
from sklearn.decomposition import IncrementalPCA, RandomizedPCA, PCA

def plot_results(X, y, label):
    plt.plot(X, y, label=label, marker='o')

def benchmark(estimator, data):
    gc.collect()
    print("Benching %s" % estimator)
    t0 = time()
    estimator.fit(data)
    training_time = time() - t0
    data_t = estimator.transform(data)
    data_r = estimator.inverse_transform(data_t)
    reconstruction_error = np.mean(np.abs(data - data_r))
    return {'time': training_time, 'error': reconstruction_error}

def plot_feature_times(all_times, batch_size, all_components, data):
    plt.figure()
    plot_results(all_components, all_times['pca'], label="PCA")
    plot_results(all_components, all_times['ipca'],
                 label="IncrementalPCA, bsize=%i" % batch_size)
    plot_results(all_components, all_times['rpca'], label="RandomizedPCA")
    plt.legend(loc="upper left")
    plt.suptitle("Algorithm runtime vs. n_components\n \
                 LFW, size %i x %i" % data.shape)
    plt.xlabel("Number of components (out of max %i)" % data.shape[1])
    plt.ylabel("Time (seconds)")

def plot_feature_errors(all_errors, batch_size, all_components, data):
    plt.figure()
    plot_results(all_components, all_errors['pca'], label="PCA")
    plot_results(all_components, all_errors['ipca'],
                 label="IncrementalPCA, bsize=%i" % batch_size)
    plot_results(all_components, all_errors['rpca'], label="RandomizedPCA")
    plt.legend(loc="lower left")
    plt.suptitle("Algorithm error vs. n_components\n"
                 "LFW, size %i x %i" % data.shape)
    plt.xlabel("Number of components (out of max %i)" % data.shape[1])
    plt.ylabel("Mean absolute error")

def plot_batch_times(all_times, n_features, all_batch_sizes, data):
    plt.figure()
    plot_results(all_batch_sizes, all_times['pca'], label="PCA")
    plot_results(all_batch_sizes, all_times['rpca'], label="RandomizedPCA")
    plot_results(all_batch_sizes, all_times['ipca'], label="IncrementalPCA")
    plt.legend(loc="lower left")
    plt.suptitle("Algorithm runtime vs. batch_size for n_components %i\n \
                 LFW, size %i x %i" % (
                 n_features, data.shape[0], data.shape[1]))
    plt.xlabel("Batch size")
    plt.ylabel("Time (seconds)")

def plot_batch_errors(all_errors, n_features, all_batch_sizes, data):
    plt.figure()
    plot_results(all_batch_sizes, all_errors['pca'], label="PCA")
    plot_results(all_batch_sizes, all_errors['ipca'], label="IncrementalPCA")
    plt.legend(loc="lower left")
    plt.suptitle("Algorithm error vs. batch_size for n_components %i\n \
                 LFW, size %i x %i" % (
                 n_features, data.shape[0], data.shape[1]))
    plt.xlabel("Batch size")
    plt.ylabel("Mean absolute error")

def fixed_batch_size_comparison(data):
    all_features = [i.astype(int) for i in np.linspace(data.shape[1] // 10,
                                                       data.shape[1], num=5)]
    batch_size = 1000
    # Compare runtimes and error for fixed batch size
    all_times = defaultdict(list)
    all_errors = defaultdict(list)
    for n_components in all_features:
        pca = PCA(n_components=n_components)
        rpca = RandomizedPCA(n_components=n_components, random_state=1999)
        ipca = IncrementalPCA(n_components=n_components, batch_size=batch_size)
        results_dict = {k: benchmark(est, data) for k, est in [('pca', pca),
                                                               ('ipca', ipca),
                                                               ('rpca', rpca)]}

        for k in sorted(results_dict.keys()):
            all_times[k].append(results_dict[k]['time'])
            all_errors[k].append(results_dict[k]['error'])

    plot_feature_times(all_times, batch_size, all_features, data)
    plot_feature_errors(all_errors, batch_size, all_features, data)

def variable_batch_size_comparison(data):
    batch_sizes = [i.astype(int) for i in np.linspace(data.shape[0] // 10,
                                                      data.shape[0], num=10)]

    for n_components in [i.astype(int) for i in
                         np.linspace(data.shape[1] // 10,
                                     data.shape[1], num=4)]:
        all_times = defaultdict(list)
        all_errors = defaultdict(list)
        pca = PCA(n_components=n_components)
        rpca = RandomizedPCA(n_components=n_components, random_state=1999)
        results_dict = {k: benchmark(est, data) for k, est in [('pca', pca),
                                                               ('rpca', rpca)]}

        # Create flat baselines to compare the variation over batch size
        all_times['pca'].extend([results_dict['pca']['time']] *
                                len(batch_sizes))
        all_errors['pca'].extend([results_dict['pca']['error']] *
                                 len(batch_sizes))
        all_times['rpca'].extend([results_dict['rpca']['time']] *
                                 len(batch_sizes))
        all_errors['rpca'].extend([results_dict['rpca']['error']] *
                                  len(batch_sizes))
        for batch_size in batch_sizes:
            ipca = IncrementalPCA(n_components=n_components,
                                  batch_size=batch_size)
            results_dict = {k: benchmark(est, data) for k, est in [('ipca',
                                                                   ipca)]}
            all_times['ipca'].append(results_dict['ipca']['time'])
            all_errors['ipca'].append(results_dict['ipca']['error'])

        plot_batch_times(all_times, n_components, batch_sizes, data)
        # RandomizedPCA error is always worse (approx 100x) than other PCA
        # tests
        plot_batch_errors(all_errors, n_components, batch_sizes, data)

faces = fetch_lfw_people(resize=.2, min_faces_per_person=5)
# limit dataset to 5000 people (don't care who they are!)
X = faces.data[:5000]
n_samples, h, w = faces.images.shape
n_features = X.shape[1]

X -= X.mean(axis=0)
X /= X.std(axis=0)

fixed_batch_size_comparison(X)
variable_batch_size_comparison(X)
plt.show()


import os
import sys
import numpy as np
import pandas as pd
import re
import io

from matplotlib.figure import Figure, AxesStack
from matplotlib.axes import Subplot

from mplstyler import StylesManager

import warnings
from . import displayobjects
from .utils import scriptdir, basedir
from IPython.core import display
from copy import deepcopy

MAGIC_TYPES = [
        # Numpy
        np.array, np.ndarray,
        # Pandas
        pd.Series, pd.DataFrame,
        Figure, Subplot,
        StylesManager,
        # View types
        displayobjects.Svg, displayobjects.Html, displayobjects.Markdown,
        display.SVG
        ]

class PathomxTool(object):
    ''' Simple wrapper class that holds the output data for a given tool; This is for user-friendliness
    not for use '''

    def __str__(self):
        return self._name

    def __repr__(self):
        return self._name

    def __init__(self, name, *args, **kwargs):
        self.__dict__.update(kwargs)
        self._name = name

def pathomx_notebook_start(vars):

    #for k, v in varsi.items():
    #    vars[k] = v

    # _keep_input_vars = ['styles']
    # vars['_pathomx_exclude_input_vars'] = [x for x in varsi.keys() if x not in _keep_input_vars]

    # Handle IO magic
    if '_io' in vars:
        for k, v in vars['_io']['input'].items():
            if v in vars:
                vars[k] = deepcopy(vars[v])
            else:
                vars[k] = None

    if '_rcParams' in vars:
        global rcParams
        from matplotlib import rcParams

        # Block warnings from deprecated rcParams here
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            for k, v in vars['_rcParams'].items():
                rcParams[k] = v

    # Legacy shim
    if '_styles' in vars:
        vars['styles'] = vars['_styles']

def pathomx_notebook_stop(vars):
    varso = {}
    if '_io' in vars:
        # Handle IO magic
        for k, v in vars['_io']['output'].items():
            if k in vars:
                vars[v] = vars[k]
            else:
                vars[v] = None

        for k, v in vars.items():
            # Check it's an accepted type for passing; and not private (starts with _)
            if not k.startswith('_') and \
                not k in vars['_io']['input'].keys():

                if type(v) in MAGIC_TYPES or k in vars['_pathomx_expected_output_vars']:
                    varso[k] = v

                elif hasattr(v, '_repr_html_'):
                    try:
                        # Check if it is a bound method (not a class definition)
                        v._repr_html_()
                    except:
                        pass
                    else:
                        varso[k] = displayobjects.Html(v)

    vars['varso'] = varso

def progress(progress):
    ''' Output the current progress to stdout on the remote core
        this will be read from stdout and displayed in the UI '''
    print("____pathomx_execute_progress_%.2f____" % progress)

class open_with_progress(io.IOBase):

    def __init__(self, f, *args, **kwargs):
        super(open_with_progress, self).__init__(f, *args, **kwargs)

        self._fsize = os.path.getsize(f)
        self._progress = None

    def read(self, *args, **kwargs):
        super(open_with_progress, self).read(*args, **kwargs)
        self.check_and_emit_progress()

    def check_and_emit_progress(self):
        # We only output at 2dp so only emit when that changes
        prg = round(self.tell() / self._fsize, 2)
        if prg != self._progress:
            self._progress = prg
            progress(prg)


"""
================================================================
Biclustering documents with the Spectral Co-clustering algorithm
================================================================

This example demonstrates the Spectral Co-clustering algorithm on the
twenty newsgroups dataset. The 'comp.os.ms-windows.misc' category is
excluded because it contains many posts containing nothing but data.

The TF-IDF vectorized posts form a word frequency matrix, which is
then biclustered using Dhillon's Spectral Co-Clustering algorithm. The
resulting document-word biclusters indicate subsets words used more
often in those subsets documents.

For a few of the best biclusters, its most common document categories
and its ten most important words get printed. The best biclusters are
determined by their normalized cut. The best words are determined by
comparing their sums inside and outside the bicluster.

For comparison, the documents are also clustered using
MiniBatchKMeans. The document clusters derived from the biclusters
achieve a better V-measure than clusters found by MiniBatchKMeans.

Output::

    Vectorizing...
    Coclustering...
    Done in 9.53s. V-measure: 0.4455
    MiniBatchKMeans...
    Done in 12.00s. V-measure: 0.3309

    Best biclusters:
    ----------------
    bicluster 0 : 1951 documents, 4373 words
    categories   : 23% talk.politics.guns, 19% talk.politics.misc, 14% sci.med
    words        : gun, guns, geb, banks, firearms, drugs, gordon, clinton, cdt, amendment

    bicluster 1 : 1165 documents, 3304 words
    categories   : 29% talk.politics.mideast, 26% soc.religion.christian, 25% alt.atheism
    words        : god, jesus, christians, atheists, kent, sin, morality, belief, resurrection, marriage

    bicluster 2 : 2219 documents, 2830 words
    categories   : 18% comp.sys.mac.hardware, 16% comp.sys.ibm.pc.hardware, 16% comp.graphics
    words        : voltage, dsp, board, receiver, circuit, shipping, packages, stereo, compression, package

    bicluster 3 : 1860 documents, 2745 words
    categories   : 26% rec.motorcycles, 23% rec.autos, 13% misc.forsale
    words        : bike, car, dod, engine, motorcycle, ride, honda, cars, bmw, bikes

    bicluster 4 : 12 documents, 155 words
    categories   : 100% rec.sport.hockey
    words        : scorer, unassisted, reichel, semak, sweeney, kovalenko, ricci, audette, momesso, nedved

"""
from __future__ import print_function

print(__doc__)

from collections import defaultdict
import operator
import re
from time import time

import numpy as np

from sklearn.cluster.bicluster import SpectralCoclustering
from sklearn.cluster import MiniBatchKMeans
from sklearn.externals import six
from sklearn.datasets.twenty_newsgroups import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.cluster import v_measure_score

def number_aware_tokenizer(doc):
    """ Tokenizer that maps all numeric tokens to a placeholder.

    For many applications, tokens that begin with a number are not directly
    useful, but the fact that such a token exists can be relevant.  By applying
    this form of dimensionality reduction, some methods may perform better.
    """
    token_pattern = re.compile(u'(?u)\\b\\w\\w+\\b')
    tokens = token_pattern.findall(doc)
    tokens = ["#NUMBER" if token[0] in "0123456789_" else token
              for token in tokens]
    return tokens

# exclude 'comp.os.ms-windows.misc'
categories = ['alt.atheism', 'comp.graphics',
              'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware',
              'comp.windows.x', 'misc.forsale', 'rec.autos',
              'rec.motorcycles', 'rec.sport.baseball',
              'rec.sport.hockey', 'sci.crypt', 'sci.electronics',
              'sci.med', 'sci.space', 'soc.religion.christian',
              'talk.politics.guns', 'talk.politics.mideast',
              'talk.politics.misc', 'talk.religion.misc']
newsgroups = fetch_20newsgroups(categories=categories)
y_true = newsgroups.target

vectorizer = TfidfVectorizer(stop_words='english', min_df=5,
                             tokenizer=number_aware_tokenizer)
cocluster = SpectralCoclustering(n_clusters=len(categories),
                                 svd_method='arpack', random_state=0)
kmeans = MiniBatchKMeans(n_clusters=len(categories), batch_size=20000,
                         random_state=0)

print("Vectorizing...")
X = vectorizer.fit_transform(newsgroups.data)

print("Coclustering...")
start_time = time()
cocluster.fit(X)
y_cocluster = cocluster.row_labels_
print("Done in {:.2f}s. V-measure: {:.4f}".format(
    time() - start_time,
    v_measure_score(y_cocluster, y_true)))

print("MiniBatchKMeans...")
start_time = time()
y_kmeans = kmeans.fit_predict(X)
print("Done in {:.2f}s. V-measure: {:.4f}".format(
    time() - start_time,
    v_measure_score(y_kmeans, y_true)))

feature_names = vectorizer.get_feature_names()
document_names = list(newsgroups.target_names[i] for i in newsgroups.target)

def bicluster_ncut(i):
    rows, cols = cocluster.get_indices(i)
    if not (np.any(rows) and np.any(cols)):
        import sys
        return sys.float_info.max
    row_complement = np.nonzero(np.logical_not(cocluster.rows_[i]))[0]
    col_complement = np.nonzero(np.logical_not(cocluster.columns_[i]))[0]
    weight = X[rows[:, np.newaxis], cols].sum()
    cut = (X[row_complement[:, np.newaxis], cols].sum() +
           X[rows[:, np.newaxis], col_complement].sum())
    return cut / weight

def most_common(d):
    """Items of a defaultdict(int) with the highest values.

    Like Counter.most_common in Python >=2.7.
    """
    return sorted(six.iteritems(d), key=operator.itemgetter(1), reverse=True)

bicluster_ncuts = list(bicluster_ncut(i)
                       for i in xrange(len(newsgroups.target_names)))
best_idx = np.argsort(bicluster_ncuts)[:5]

print()
print("Best biclusters:")
print("----------------")
for idx, cluster in enumerate(best_idx):
    n_rows, n_cols = cocluster.get_shape(cluster)
    cluster_docs, cluster_words = cocluster.get_indices(cluster)
    if not len(cluster_docs) or not len(cluster_words):
        continue

    # categories
    counter = defaultdict(int)
    for i in cluster_docs:
        counter[document_names[i]] += 1
    cat_string = ", ".join("{:.0f}% {}".format(float(c) / n_rows * 100, name)
                           for name, c in most_common(counter)[:3])

    # words
    out_of_cluster_docs = cocluster.row_labels_ != cluster
    out_of_cluster_docs = np.where(out_of_cluster_docs)[0]
    word_col = X[:, cluster_words]
    word_scores = np.array(word_col[cluster_docs, :].sum(axis=0) -
                           word_col[out_of_cluster_docs, :].sum(axis=0))
    word_scores = word_scores.ravel()
    important_words = list(feature_names[cluster_words[i]]
                           for i in word_scores.argsort()[:-11:-1])

    print("bicluster {} : {} documents, {} words".format(
        idx, n_rows, n_cols))
    print("categories   : {}".format(cat_string))
    print("words        : {}\n".format(', '.join(important_words)))


#reading different model files

import numpy as np
from numpy import recfromtxt, genfromtxt
import pandas as pd
from astropy import units as u

import logging
# Adding logging support
logger = logging.getLogger(__name__)

from tardis.util import parse_quantity

class ConfigurationError(Exception):
    pass

def read_density_file(density_filename, density_filetype, time_explosion, v_inner_boundary=0.0, v_outer_boundary=np.inf):
    """
    read different density file formats

    Parameters
    ----------

    density_filename: ~str
        filename or path of the density file

    density_filetype: ~str
        type of the density file

    time_explosion: ~astropy.units.Quantity
        time since explosion used to scale the density

    """
    file_parsers = {'artis': read_artis_density,
                    'simple_ascii': read_simple_ascii_density}

    time_of_model, index, v_inner, v_outer, unscaled_mean_densities = file_parsers[density_filetype](density_filename)
    mean_densities = calculate_density_after_time(unscaled_mean_densities, time_of_model, time_explosion)

    if v_inner_boundary > v_outer_boundary:
        raise ConfigurationError('v_inner_boundary > v_outer_boundary '
                                 '({0:s} > {1:s}). unphysical!'.format(
            v_inner_boundary, v_outer_boundary))

    if (not np.isclose(v_inner_boundary, 0.0 * u.km / u.s,
                       atol=1e-8 * u.km / u.s)
        and v_inner_boundary > v_inner[0]):

        if v_inner_boundary > v_outer[-1]:
            raise ConfigurationError('Inner boundary selected outside of model')

        inner_boundary_index = v_inner.searchsorted(v_inner_boundary) - 1

    else:
        inner_boundary_index = None
        v_inner_boundary = v_inner[0]
        logger.warning("v_inner_boundary requested too small for readin file."
                       " Boundary shifted to match file.")

    if not np.isinf(v_outer_boundary) and v_outer_boundary < v_outer[-1]:
        outer_boundary_index = v_outer.searchsorted(v_outer_boundary) + 1
    else:
        outer_boundary_index = None
        v_outer_boundary = v_outer[-1]
        logger.warning("v_outer_boundary requested too large for readin file. Boundary shifted to match file.")

    v_inner = v_inner[inner_boundary_index:outer_boundary_index]
    v_inner[0] = v_inner_boundary

    v_outer = v_outer[inner_boundary_index:outer_boundary_index]
    v_outer[-1] = v_outer_boundary

    mean_densities = mean_densities[inner_boundary_index:outer_boundary_index]

    return v_inner, v_outer, mean_densities, inner_boundary_index, outer_boundary_index

def read_abundances_file(abundance_filename, abundance_filetype, inner_boundary_index=None, outer_boundary_index=None):
    """
    read different density file formats

    Parameters
    ----------

    abundance_filename: ~str
        filename or path of the density file

    abundance_filetype: ~str
        type of the density file

    inner_boundary_index: int
        index of the inner shell, default None

    outer_boundary_index: int
        index of the outer shell, default None

    """

    file_parsers = {'simple_ascii': read_simple_ascii_abundances,
                    'artis': read_simple_ascii_abundances}

    index, abundances = file_parsers[abundance_filetype](abundance_filename)
    if outer_boundary_index is not None:
        outer_boundary_index_m1 = outer_boundary_index - 1
    else:
        outer_boundary_index_m1 = None
    index = index[inner_boundary_index:outer_boundary_index]
    abundances = abundances.ix[:, slice(inner_boundary_index, outer_boundary_index_m1)]
    abundances.columns = np.arange(len(abundances.columns))
    return index, abundances

def read_simple_ascii_density(fname):
    """
    Reading a density file of the following structure (example; lines starting with a hash will be ignored):
    The first density describes the mean density in the center of the model and is not used.
    5 s
    #index velocity [km/s] density [g/cm^3]
    0 1.1e4 1.6e8
    1 1.2e4 1.7e8

    Parameters
    ----------

    fname: str
        filename or path with filename

    Returns
    -------

    time_of_model: ~astropy.units.Quantity
        time at which the model is valid

    data: ~pandas.DataFrame
        data frame containing index, velocity (in km/s) and density
    """

    with open(fname) as fh:
        time_of_model_string = fh.readline().strip()
        time_of_model = parse_quantity(time_of_model_string)

    data = recfromtxt(fname, skip_header=1, names=('index', 'velocity', 'density'), dtype=(int, float, float))
    velocity = (data['velocity'] * u.km / u.s).to('cm/s')
    v_inner, v_outer = velocity[:-1], velocity[1:]
    mean_density = (data['density'] * u.Unit('g/cm^3'))[1:]

    return time_of_model, data['index'], v_inner, v_outer, mean_density

def read_artis_density(fname):
    """
    Reading a density file of the following structure (example; lines starting with a hash will be ignored):
    The first density describes the mean density in the center of the model and is not used.
    5
    #index velocity [km/s] log10(density) [log10(g/cm^3)]
    0 1.1e4 1.6e8
    1 1.2e4 1.7e8

    Parameters
    ----------

    fname: str
        filename or path with filename

    Returns
    -------

    time_of_model: ~astropy.units.Quantity
        time at which the model is valid

    data: ~pandas.DataFrame
        data frame containing index, velocity (in km/s) and density
    """

    with open(fname) as fh:
        for i, line in enumerate(file(fname)):
            if i == 0:
                no_of_shells = np.int64(line.strip())
            elif i == 1:
                time_of_model = u.Quantity(float(line.strip()), 'day').to('s')
            elif i == 2:
                break

    artis_model_columns = ['index', 'velocities', 'mean_densities_0', 'ni56_fraction', 'co56_fraction', 'fe52_fraction',
                           'cr48_fraction']
    artis_model = recfromtxt(fname, skip_header=2, usecols=(0, 1, 2, 4, 5, 6, 7), unpack=True,
                                dtype=[(item, np.float64) for item in artis_model_columns])

    velocity = u.Quantity(artis_model['velocities'], 'km/s').to('cm/s')
    mean_density = u.Quantity(10 ** artis_model['mean_densities_0'], 'g/cm^3')
    v_inner, v_outer = velocity[:-1], velocity[1:]

    return time_of_model, artis_model['index'], v_inner, v_outer, mean_density

def read_simple_ascii_abundances(fname):
    """
    Reading an abundance file of the following structure (example; lines starting with hash will be ignored):
    The first line of abundances describe the abundances in the center of the model and are not used.
    #index element1, element2, ..., element30
    0 0.4 0.3, .. 0.2

    Parameters
    ----------

    fname: str
        filename or path with filename

    Returns
    -------

    index: ~np.ndarray
        containing the indices

    abundances: ~pandas.DataFrame
        data frame containing index, element1 - element30 and columns according to the shells
    """
    data = np.loadtxt(fname)

    index = data[1:,0].astype(int)
    abundances = pd.DataFrame(data[1:,1:].transpose(), index=np.arange(1, data.shape[1]))

    return index, abundances

def calculate_density_after_time(densities, time_0, time_explosion):
    """
    scale the density from an initial time of the model to the time of the explosion by ^-3

    Parameters:
    -----------

    densities: ~astropy.units.Quantity
        densities

    time_0: ~astropy.units.Quantity
        time of the model

    time_explosion: ~astropy.units.Quantity
        time to be scaled to

    Returns:
    --------

    scaled_density
    """

    return densities * (time_explosion / time_0) ** -3


from itertools import product
import numpy as np
from numpy.testing import assert_almost_equal, assert_array_almost_equal

from sklearn import datasets
from sklearn import manifold
from sklearn import neighbors
from sklearn import pipeline
from sklearn import preprocessing
from sklearn.utils.testing import assert_less

eigen_solvers = ['auto', 'dense', 'arpack']
path_methods = ['auto', 'FW', 'D']

def test_isomap_simple_grid():
    # Isomap should preserve distances when all neighbors are used
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # distances from each point to all others
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()
            assert_array_almost_equal(G, G_iso)

def test_isomap_reconstruction_error():
    # Same setup as in test_isomap_simple_grid, with an added dimension
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # add noise in a third dimension
    rng = np.random.RandomState(0)
    noise = 0.1 * rng.randn(Npts, 1)
    X = np.concatenate((X, noise), 1)

    # compute input kernel
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    centerer = preprocessing.KernelCenterer()
    K = centerer.fit_transform(-0.5 * G ** 2)

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            # compute output kernel
            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()

            K_iso = centerer.fit_transform(-0.5 * G_iso ** 2)

            # make sure error agrees
            reconstruction_error = np.linalg.norm(K - K_iso) / Npts
            assert_almost_equal(reconstruction_error,
                                clf.reconstruction_error())

def test_transform():
    n_samples = 200
    n_components = 10
    noise_scale = 0.01

    # Create S-curve dataset
    X, y = datasets.samples_generator.make_s_curve(n_samples, random_state=0)

    # Compute isomap embedding
    iso = manifold.Isomap(n_components, 2)
    X_iso = iso.fit_transform(X)

    # Re-embed a noisy version of the points
    rng = np.random.RandomState(0)
    noise = noise_scale * rng.randn(*X.shape)
    X_iso2 = iso.transform(X + noise)

    # Make sure the rms error on re-embedding is comparable to noise_scale
    assert_less(np.sqrt(np.mean((X_iso - X_iso2) ** 2)), 2 * noise_scale)

def test_pipeline():
    # check that Isomap works fine as a transformer in a Pipeline
    # only checks that no error is raised.
    # TODO check that it actually does something useful
    X, y = datasets.make_blobs(random_state=0)
    clf = pipeline.Pipeline(
        [('isomap', manifold.Isomap()),
         ('clf', neighbors.KNeighborsClassifier())])
    clf.fit(X, y)
    assert_less(.9, clf.score(X, y))


from pyomo.environ import *
import numpy as np
import util
import config as cfg
import pdb
import pandas as pd
import copy
import dispatch_budget
import logging

def surplus_capacity(model):
    return model.surplus_capacity + model.peak_penalty * model.weight_on_peak_penalty

def define_penalty_to_preference_high_cost_gen_maint_during_peak(model):
    # if forced to choose between having high cost or low cost gen be on maintenance when load is high, we'd rather high cost gen be doing maintenance
    # this should lower production cost overall and make maintenance schedules less random
    return model.peak_penalty == sum([sum([model.marginal_costs[g]*model.max_load_by_group[i]*model.scheduled_maintenance[i, g] for g in model.g])
                                      for i in model.i])

def feasible_maintenance_constraint_0(model, i, g):
    return model.scheduled_maintenance[i, g] >= 0

def feasible_maintenance_constraint_1(model, i, g):
    return model.scheduled_maintenance[i, g] <= 1

def define_available_gen(model, i):
    return model.available_gen[i] == sum([(1 - model.scheduled_maintenance[i, g]) * model.pmax[g] for g in model.g])

def meet_maintenance_constraint(model, g):
    # average maintenance across the hours == annual maintenance rate
    return sum([model.scheduled_maintenance[i, g] * model.group_lengths[i] for i in model.i]) == model.annual_maintenace_hours[g]

def define_surplus_capacity(model, i):
    return model.surplus_capacity >= model.available_gen[i] - model.max_load_by_group[i]

def scale_load_to_system(load, pmaxs, typical_reserve=1.15):
    max_load = load.max()
    sum_cap = sum(pmaxs)
    if (max_load * typical_reserve) > sum_cap:
        assert max_load != 0
        load2 = load * (sum_cap / (max_load * typical_reserve))
        return load2
    else:
        return load

def schedule_generator_maintenance(load, pmaxs, annual_maintenance_rates, dispatch_periods, marginal_costs, print_opt=False):
    # annual maintenance rates must be between zero and one
    annual_maintenance_rates = np.clip(annual_maintenance_rates, 0, 1)

    # gives the index for the change between dispatch_periods
    group_cuts = list(np.where(np.diff(dispatch_periods) != 0)[0] + 1) if dispatch_periods is not None else None
    group_lengths = np.array([group_cuts[0]] + list(np.diff(group_cuts)) + [len(load) - group_cuts[-1]])
    num_groups = len(group_cuts) + 1

    # necessary to scale load in some cases for the optimization to work. Basically, load shouldn't be > gen
    load_scaled = scale_load_to_system(load, pmaxs)
    max_load_by_group = np.array([np.max(ls) for ls in np.array_split(load_scaled, np.array(group_cuts))])

    annual_maintenace_hours = annual_maintenance_rates*len(load)

    pmaxs_zero = np.nonzero(pmaxs==0)[0]
    pmaxs_not_zero = np.nonzero(pmaxs)[0]

    estimated_peak_penalty = sum(sum(np.outer(marginal_costs[pmaxs_not_zero],max_load_by_group).T*annual_maintenance_rates[pmaxs_not_zero]))
    estimated_surplus_capacity = (pmaxs.sum() - max_load_by_group.min())*(1-annual_maintenance_rates.mean())
    weight_on_peak_penalty = estimated_surplus_capacity/estimated_peak_penalty/10.

    model = ConcreteModel()

    # INPUT PARAMS
    model.i = RangeSet(0, num_groups - 1)
    model.g = RangeSet(0, len(pmaxs_not_zero) - 1)
    model.annual_maintenace_hours = Param(model.g, initialize=dict(zip(model.g.keys(), annual_maintenace_hours[pmaxs_not_zero])))
    model.pmax = Param(model.g, initialize=dict(zip(model.g.keys(), pmaxs[pmaxs_not_zero])))
    model.marginal_costs = Param(model.g, initialize=dict(zip(model.g.keys(), marginal_costs[pmaxs_not_zero])))
    model.max_load_by_group = Param(model.i, initialize=dict(zip(model.i.keys(), max_load_by_group)))
    model.group_lengths = Param(model.i, initialize=dict(zip(model.i.keys(), group_lengths)))
    model.weight_on_peak_penalty = Param(default=weight_on_peak_penalty)

    # DECISIONS VARIABLES
    model.available_gen = Var(model.i, within=NonNegativeReals)
    model.scheduled_maintenance = Var(model.i, model.g, within=NonNegativeReals)
    model.surplus_capacity = Var(within=NonNegativeReals)
    model.peak_penalty = Var(within=NonNegativeReals)

    # CONSTRAINTS
    model.define_available_gen = Constraint(model.i, rule=define_available_gen)
    model.feasible_maintenance_constraint_0 = Constraint(model.i, model.g, rule=feasible_maintenance_constraint_0)
    model.feasible_maintenance_constraint_1 = Constraint(model.i, model.g, rule=feasible_maintenance_constraint_1)
    model.meet_maintenance_constraint = Constraint(model.g, rule=meet_maintenance_constraint)
    model.define_surplus_capacity = Constraint(model.i, rule=define_surplus_capacity)
    model.define_penalty_to_preference_high_cost_gen_maint_during_peak = Constraint(rule=define_penalty_to_preference_high_cost_gen_maint_during_peak)

    # OBJECTIVE
    model.objective = Objective(rule=surplus_capacity, sense=minimize)

    # SOLVE AND EXPORT RESULTS
    solver = SolverFactory(cfg.solver_name or "cbc") # use cbc by default for testing, when you import config in a test, solver_name is None
    results = solver.solve(model, tee=print_opt)
    model.solutions.load_from(results)

    scheduled_maintenance = np.empty((num_groups, len(pmaxs)))
    scheduled_maintenance[:, pmaxs_zero] = annual_maintenance_rates[pmaxs_zero]
    scheduled_maintenance[:, pmaxs_not_zero] = np.array([[model.scheduled_maintenance[i, g].value for i in model.i.keys()] for g in model.g.keys()]).T
    return scheduled_maintenance

def schedule_generator_maintenance_loop(load, pmaxs, annual_maintenance_rates, dispatch_periods, scheduling_order):
    # if nothing else, better to schedule the large generators first
    scheduling_order = np.argsort(-pmaxs) if scheduling_order is None else scheduling_order

    # annual maintenance rates must be between zero and one
    annual_maintenance_rates = np.clip(annual_maintenance_rates, 0, 1)

    # gives the index for the change between dispatch_periods
    group_cuts = list(np.where(np.diff(dispatch_periods) != 0)[0] + 1) if dispatch_periods is not None else None
    group_lengths = np.array([group_cuts[0]] + list(np.diff(group_cuts)) + [len(load) - group_cuts[-1]])
    num_groups = len(group_cuts) + 1

    # necessary to scale load in some cases for the optimization to work. Basically, load shouldn't be > gen
    load_scaled = scale_load_to_system(load, pmaxs)
    load_scaled = np.concatenate([[np.max(ls)]*gl for gl, ls in zip(group_lengths, np.array_split(load_scaled, np.array(group_cuts)))])

    pmaxs_clipped = copy.deepcopy(pmaxs)
    pmaxs_clipped = np.clip(pmaxs_clipped, 1e-1, None)
    maintenance_energy = annual_maintenance_rates*pmaxs_clipped*len(load)
    scheduled_maintenance = np.zeros((num_groups, len(pmaxs)))

    # loop through and schedule maintenance for each generator one at a time. Update the net load after each one.
    for i in scheduling_order:
        energy_allocation = dispatch_budget.dispatch_to_energy_budget(load_scaled, -maintenance_energy[i], pmins=0, pmaxs=pmaxs_clipped[i])
        scheduled_maintenance[:, i] = np.clip(np.array([np.mean(ls) for ls in np.array_split(energy_allocation, np.array(group_cuts))])/pmaxs_clipped[i], 0, 1)
        load_scaled += np.concatenate([[sm * pmaxs[i]]*gl for gl, sm in zip(group_lengths, scheduled_maintenance[:, i])])

    if not all(np.isclose(annual_maintenance_rates, (scheduled_maintenance.T * group_lengths).sum(axis=1)/len(load))):
        logging.warning("scheduled maintance rates don't all match the annual maintenance rates")
    return scheduled_maintenance


'''
AUDIO CLASSICAL COMPOSER IDENTIFICATION BASED ON:
A SPECTRAL BANDWISE FEATURE-BASED SYSTEM
'''
import essentia
from essentia.standard import *
import glob
import numpy as np
import arff
from scipy import stats
import collections
import cv2
import matplotlib
import matplotlib.pyplot as plt

#### gabor filters

def build_filters():
	 filters = []
	 ksize = 31
	 for theta in np.arange(0, np.pi, np.pi / 16):
		 kern = cv2.getGaborKernel((ksize, ksize), 4.0, theta, 10.0, 0.5, 0, ktype=cv2.CV_32F)
		 kern /= 1.5*kern.sum()
		 filters.append(kern)
	 return filters

def process(img, filters):
	 accum = np.zeros_like(img)
	 for kern in filters:
		 fimg = cv2.filter2D(img, cv2.CV_8UC3, kern)
		 np.maximum(accum, fimg, accum)
	 return accum

###

# Dataset creation with specific attributes (spectral features) and a specific class (composer's name)

'''
Audio files trasformed into the frequency domain through a 1024-sample STFT with 50% overlap.
The spectrum is divided into 50 mel-spaced bands.
'''

dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/bach/*.wav")
fft = FFT()
melbands = MelBands(numberBands = 50)

flatness = FlatnessDB()
rolloff = RollOff()
centroid = SpectralCentroidTime()
flux = Flux()
energy = EnergyBand()
zero = ZeroCrossingRate()
spectrum = Spectrum()
w = Windowing(type = 'hann')
mfcc = MFCC()
silence = SilenceRate(thresholds = [0.01])

f = open('definitive_train.txt', 'wb')
f.write('@RELATION "composer dataset"\n')
f.write('\n')
f.write('@ATTRIBUTE filename STRING\n')
f.write('@ATTRIBUTE MFCC-0 REAL\n')
f.write('@ATTRIBUTE MFCC-1 REAL\n')
f.write('@ATTRIBUTE MFCC-2 REAL\n')
f.write('@ATTRIBUTE MFCC-3 REAL\n')
f.write('@ATTRIBUTE MFCC-4 REAL\n')
f.write('@ATTRIBUTE MFCC-5 REAL\n')
f.write('@ATTRIBUTE MFCC-6 REAL\n')
f.write('@ATTRIBUTE MFCC-7 REAL\n')
f.write('@ATTRIBUTE MFCC-8 REAL\n')
f.write('@ATTRIBUTE MFCC-9 REAL\n')
f.write('@ATTRIBUTE MFCC-10 REAL\n')
f.write('@ATTRIBUTE MFCC-11 REAL\n')
f.write('@ATTRIBUTE MFCC-12 REAL\n')
f.write('@ATTRIBUTE flatness-mean REAL\n')
f.write('@ATTRIBUTE flatness-variance REAL\n')
f.write('@ATTRIBUTE rolloff-mean REAL\n')
f.write('@ATTRIBUTE rolloff-variance REAL\n')
f.write('@ATTRIBUTE centroid-mean REAL\n')
f.write('@ATTRIBUTE centroid-variance REAL\n')
f.write('@ATTRIBUTE flux-mean REAL\n')
f.write('@ATTRIBUTE flux-variance REAL\n')
f.write('@ATTRIBUTE energy-mean REAL\n')
f.write('@ATTRIBUTE energy-variance REAL\n')
f.write('@ATTRIBUTE ZCR-mean REAL\n')
f.write('@ATTRIBUTE ZCR-variance REAL\n')
f.write('@ATTRIBUTE flatness-std REAL\n')
f.write('@ATTRIBUTE flatness-hmean REAL\n')
f.write('@ATTRIBUTE silences REAL\n')
f.write('@ATTRIBUTE gaborfilter-mean REAL\n')
f.write('@ATTRIBUTE gaborfilter-variance REAL\n')
f.write('@ATTRIBUTE composer {bach, beethoven, chopin, haydn, liszt, mendelssohn, mozart, vivaldi}\n')
f.write('\n')
f.write('@DATA\n')

dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/bach'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'bach'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))

			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('bach')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1
# 2

dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/beethoven/*.wav")
dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/beethoven'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'beethoven'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))

			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('beethoven')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1
# 3

dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/chopin/*.wav")
dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/chopin'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'chopin'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))

			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('chopin')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1
# 4
dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/haydn/*.wav")
dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/haydn'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'haydn'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))

			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('haydn')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1
'''
# 5
dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/liszt/*.wav")
dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/liszt'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'liszt'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))
			'''
			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('liszt')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1

# 6
dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/mendelssohn/*.wav")
dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/mendelssohn'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'mendelssohn'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))

			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('mendelssohn')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1
# 7
dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/mozart/*.wav")
dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/mozart'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'mozart'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))

			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('mozart')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1
# 8
dirList = glob.glob("/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/datasets/vivaldi/*.wav")
dirimg = '/home/usuario/Escritorio/SMC/2 term/Music Information Retrieval/Classical Composer Identification/code_10/pictures/vivaldi'
dirname = str(dirimg) +'/*.png'
piclist = glob.glob(dirname)
counter = 0
for audio_file in dirList:

	# Selecting the expectrogram
	for item in piclist:

		if item.split('/')[-1].split('.')[0] == audio_file.split('/')[-1].split('.')[0]:

			picname = str(dirimg)+'/'+str(audio_file.split('/')[-1].split('.')[0]) + '.png'
			flat = []
			rol = []
			cen = []
			flu = []
			ene = []
			zer = []
			mfccs = []
			stft = []
			sil = []
			mean_counter = []

			# Loading audio
			audio = MonoLoader(filename = audio_file)()

			# Features extraction
			for frame in FrameGenerator(audio, frameSize = 1024, hopSize = 512, startFromZero=True):

				bands = melbands(spectrum(frame))
				stft.append(fft(frame))
				flat.append(flatness(bands))
				rol.append(rolloff(bands))
				cen.append(centroid(bands))
				flu.append(flux(bands))
				ene.append(energy(bands))
				zer.append(zero(frame))
				mfcc_bands, mfcc_coeffs = mfcc(spectrum(w(frame)))
				mfccs.append(mfcc_coeffs)
				sil.append(silence(frame))
			rate = collections.Counter()
			rate.update(sil)
			rate = rate.most_common(1)

			composer = 'vivaldi'

			# Gabor filter analysis

			if __name__ == '__main__':
				import sys
			print __doc__
			try:
				img_fn = sys.argv[1]
			except:
				img_fn = picname
			img = cv2.imread(img_fn)
			if img is None:
				print 'Failed to load image file:', img_fn
				sys.exit(1)
			filters = build_filters()
			res1 = process(img, filters)
			for i in range(len(res1)-1):
				for j in range(len(res1[i])-1):
					mean_counter.append(np.mean(res1[i][j]))

			f.write('%s' %audio_file.split('/')[-1].split('.')[0].split('vivaldi')[0])
			f.write(',')
			f.write('%r' %np.mean(mfccs[0]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[1]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[2]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[3]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[4]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[5]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[6]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[7]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[8]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[9]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[10]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[11]))
			f.write(',')
			f.write('%r' %np.mean(mfccs[12]))
			f.write(',')
			f.write('%r' %np.mean(flat))
			f.write(',')
			f.write('%r' %np.var(flat))
			f.write(',')
			f.write('%r' %np.mean(rol))
			f.write(',')
			f.write('%r' %np.var(rol))
			f.write(',')
			f.write('%r' %np.mean(cen))
			f.write(',')
			f.write('%r' %np.var(cen))
			f.write(',')
			f.write('%r' %np.mean(flu))
			f.write(',')
			f.write('%r' %np.var(flu))
			f.write(',')
			f.write('%r' %np.mean(ene))
			f.write(',')
			f.write('%r' %np.var(ene))
			f.write(',')
			f.write('%r' %np.mean(zer))
			f.write(',')
			f.write('%r' %np.var(zer))
			f.write(',')
			f.write('%r' %np.std(flat))
			f.write(',')
			f.write('%r' %stats.hmean(flat))
			f.write(',')
			f.write('%r' %rate[0][1])
			f.write(',')
			f.write('%r' %np.var(mean_counter))
			f.write(',')
			f.write('%r' %np.std(mean_counter))
			f.write(',')
			f.write('%s' %composer)
			f.write('\n')

			counter += 1

f.write('%\n')
f.write('%\n')
f.write('%\n')

f.close()


import itertools
import unittest

import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal, \
     assert_raises
from scipy import stats
from sklearn import mixture
from sklearn.datasets.samples_generator import make_spd_matrix

rng = np.random.RandomState(0)

def test_sample_gaussian():
    """
    Test sample generation from mixture.sample_gaussian where covariance
    is diagonal, spherical and full
    """

    n_features, n_samples = 2, 300
    axis = 1
    mu = rng.randint(10) * rng.rand(n_features)
    cv = (rng.rand(n_features) + 1.0) ** 2

    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='diag', n_samples=n_samples)

    assert np.allclose(samples.mean(axis), mu, atol=1.3)
    assert np.allclose(samples.var(axis), cv, atol=1.5)

    # the same for spherical covariances
    cv = (rng.rand() + 1.0) ** 2
    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='spherical', n_samples=n_samples)

    assert np.allclose(samples.mean(axis), mu, atol=1.5)
    assert np.allclose(
        samples.var(axis), np.repeat(cv, n_features), atol=1.5)

    # and for full covariances
    A = rng.randn(n_features, n_features)
    cv = np.dot(A.T, A) + np.eye(n_features)
    samples = mixture.sample_gaussian(
        mu, cv, covariance_type='full', n_samples=n_samples)
    assert np.allclose(samples.mean(axis), mu, atol=1.3)
    assert np.allclose(np.cov(samples), cv, atol=2.5)

def _naive_lmvnpdf_diag(X, mu, cv):
    # slow and naive implementation of lmvnpdf
    ref = np.empty((len(X), len(mu)))
    stds = np.sqrt(cv)
    for i, (m, std) in enumerate(itertools.izip(mu, stds)):
        ref[:, i] = np.log(stats.norm.pdf(X, m, std)).sum(axis=1)
    return ref

def test_lmvnpdf_diag():
    """
    test a slow and naive implementation of lmvnpdf and
    compare it to the vectorized version (mixture.lmvnpdf) to test
    for correctness
    """
    n_features, n_components, n_samples = 2, 3, 10
    mu = rng.randint(10) * rng.rand(n_components, n_features)
    cv = (rng.rand(n_components, n_features) + 1.0) ** 2
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    ref = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = mixture.log_multivariate_normal_density(X, mu, cv, 'diag')
    assert_array_almost_equal(lpr, ref)

def test_lmvnpdf_spherical():
    n_features, n_components, n_samples = 2, 3, 10

    mu = rng.randint(10) * rng.rand(n_components, n_features)
    spherecv = rng.rand(n_components, 1) ** 2 + 1
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    cv = np.tile(spherecv, (n_features, 1))
    reference = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = mixture.log_multivariate_normal_density(X, mu, spherecv,
                                                  'spherical')
    assert_array_almost_equal(lpr, reference)

def test_lmvnpdf_full():
    n_features, n_components, n_samples = 2, 3, 10

    mu = rng.randint(10) * rng.rand(n_components, n_features)
    cv = (rng.rand(n_components, n_features) + 1.0) ** 2
    X = rng.randint(10) * rng.rand(n_samples, n_features)

    fullcv = np.array([np.diag(x) for x in cv])

    reference = _naive_lmvnpdf_diag(X, mu, cv)
    lpr = mixture.log_multivariate_normal_density(X, mu, fullcv, 'full')
    assert_array_almost_equal(lpr, reference)

def test_GMM_attributes():
    n_components, n_features = 10, 4
    covariance_type = 'diag'
    g = mixture.GMM(n_components, covariance_type, random_state=rng)
    weights = rng.rand(n_components)
    weights = weights / weights.sum()
    means = rng.randint(-20, 20, (n_components, n_features))

    assert g.n_components == n_components
    assert g.covariance_type == covariance_type

    g.weights_ = weights
    assert_array_almost_equal(g.weights_, weights)
    g.means_ = means
    assert_array_almost_equal(g.means_, means)

    covars = (0.1 + 2 * rng.rand(n_components, n_features)) ** 2
    g.covars_ = covars
    assert_array_almost_equal(g.covars_, covars)
    assert_raises(ValueError, g._set_covars, [])
    assert_raises(ValueError, g._set_covars,
                  np.zeros((n_components - 2, n_features)))

    assert_raises(ValueError, mixture.GMM, n_components=20,
                  covariance_type='badcovariance_type')

class GMMTester():
    do_test_eval = True

    def _setUp(self):
        self.n_components = 10
        self.n_features = 4
        self.weights = rng.rand(self.n_components)
        self.weights = self.weights / self.weights.sum()
        self.means = rng.randint(-20, 20, (self.n_components, self.n_features))
        self.threshold = -0.5
        self.I = np.eye(self.n_features)
        self.covars = {'spherical': (0.1 + 2 * \
                        rng.rand(self.n_components, self.n_features)) ** 2,
                  'tied': make_spd_matrix(self.n_features, random_state=0) +\
                        5 * self.I,
                  'diag': (0.1 + 2 * rng.rand(self.n_components,\
                        self.n_features)) ** 2,
                  'full': np.array([make_spd_matrix(self.n_features,\
                        random_state=0)
                      + 5 * self.I for x in range(self.n_components)])}

    def test_eval(self):
        if not self.do_test_eval:
            return  # DPGMM does not support setting the means and
        # covariances before fitting There is no way of fixing this
        # due to the variational parameters being more expressive than
        # covariance matrices
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type, random_state=rng)
        # Make sure the means are far apart so responsibilities.argmax()
        # picks the actual component used to generate the observations.
        g.means_ = 20 * self.means
        g.covars_ = self.covars[self.covariance_type]
        g.weights_ = self.weights

        gaussidx = np.repeat(range(self.n_components), 5)
        n_samples = len(gaussidx)
        X = rng.randn(n_samples, self.n_features) + g.means_[gaussidx]

        ll, responsibilities = g.eval(X)

        self.assertEqual(len(ll), n_samples)
        self.assertEqual(responsibilities.shape,
                         (n_samples, self.n_components))
        assert_array_almost_equal(responsibilities.sum(axis=1),
                                  np.ones(n_samples))
        assert_array_equal(responsibilities.argmax(axis=1), gaussidx)

    def test_sample(self, n=100):
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type, random_state=rng)
        # Make sure the means are far apart so responsibilities.argmax()
        # picks the actual component used to generate the observations.
        g.means_ = 20 * self.means
        g.covars_ = np.maximum(self.covars[self.covariance_type], 0.1)
        g.weights_ = self.weights

        samples = g.sample(n)
        self.assertEquals(samples.shape, (n, self.n_features))

    def test_train(self, params='wmc'):
        g = mixture.GMM(n_components=self.n_components,
                        covariance_type=self.covariance_type)
        g.weights_ = self.weights
        g.means_ = self.means
        g.covars_ = 20 * self.covars[self.covariance_type]

        # Create a training set by sampling from the predefined distribution.
        X = g.sample(n_samples=100)
        g = self.model(n_components=self.n_components,
                       covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-1,
                       n_iter=1, init_params=params)
        g.fit(X)

        # Do one training iteration at a time so we can keep track of
        # the log likelihood to make sure that it increases after each
        # iteration.
        trainll = []
        for iter in xrange(5):
            g.params = params
            g.init_params = ''
            g.fit(X)
            trainll.append(self.score(g, X))
        g.n_iter = 10
        g.init_params = ''
        g.params = params
        g.fit(X)  # finish fitting

        # Note that the log likelihood will sometimes decrease by a
        # very small amount after it has more or less converged due to
        # the addition of min_covar to the covariance (to prevent
        # underflow).  This is why the threshold is set to -0.5
        # instead of 0.
        delta_min = np.diff(trainll).min()
        self.assertTrue(
            delta_min > self.threshold,
            "The min nll increase is %f which is lower than the admissible"
            " threshold of %f, for model %s. The likelihoods are %s."
            % (delta_min, self.threshold, self.covariance_type, trainll))

    def test_train_degenerate(self, params='wmc'):
        """ Train on degenerate data with 0 in some dimensions
        """
        # Create a training set by sampling from the predefined distribution.
        X = rng.randn(100, self.n_features)
        X.T[1:] = 0
        g = self.model(n_components=2, covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-3, n_iter=5,
                       init_params=params)
        g.fit(X)
        trainll = g.score(X)
        self.assertTrue(np.sum(np.abs(trainll / 100 / X.shape[1])) < 5)

    def test_train_1d(self, params='wmc'):
        """ Train on 1-D data
        """
        # Create a training set by sampling from the predefined distribution.
        X = rng.randn(100, 1)
        #X.T[1:] = 0
        g = self.model(n_components=2, covariance_type=self.covariance_type,
                       random_state=rng, min_covar=1e-7, n_iter=5,
                       init_params=params)
        g.fit(X)
        trainll = g.score(X)
        if isinstance(g, mixture.DPGMM):
            self.assertTrue(np.sum(np.abs(trainll / 100)) < 5)
        else:
            self.assertTrue(np.sum(np.abs(trainll / 100)) < 2)

    def score(self, g, X):
        return g.score(X).sum()

class TestGMMWithSphericalCovars(unittest.TestCase, GMMTester):
    covariance_type = 'spherical'
    model = mixture.GMM
    setUp = GMMTester._setUp

class TestGMMWithDiagonalCovars(unittest.TestCase, GMMTester):
    covariance_type = 'diag'
    model = mixture.GMM
    setUp = GMMTester._setUp

class TestGMMWithTiedCovars(unittest.TestCase, GMMTester):
    covariance_type = 'tied'
    model = mixture.GMM
    setUp = GMMTester._setUp

class TestGMMWithFullCovars(unittest.TestCase, GMMTester):
    covariance_type = 'full'
    model = mixture.GMM
    setUp = GMMTester._setUp

def test_multiple_init():
    """Test that multiple inits does not much worse than a single one"""
    X = rng.randn(30, 5)
    X[:10] += 2
    g = mixture.GMM(n_components=2, covariance_type='spherical',
                    random_state=rng, min_covar=1e-7, n_iter=5)
    train1 = g.fit(X).score(X).sum()
    g.n_init = 5
    train2 = g.fit(X).score(X).sum()
    assert train2 >= train1 - 1.e-2

def test_n_parameters():
    """Test that the right number of parameters is estimated"""
    n_samples, n_dim, n_components = 7, 5, 2
    X = rng.randn(n_samples, n_dim)
    n_params = {'spherical': 13, 'diag': 21, 'tied': 26, 'full': 41}
    for cv_type in ['full', 'tied', 'diag', 'spherical']:
        g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                        random_state=rng, min_covar=1e-7, n_iter=1)
        g.fit(X)
        assert g._n_parameters() == n_params[cv_type]

def test_aic():
    """ Test the aic and bic criteria"""
    n_samples, n_dim, n_components = 50, 3, 2
    X = rng.randn(n_samples, n_dim)
    SGH = 0.5 * (X.var() + np.log(2 * np.pi))  # standard gaussian entropy

    for cv_type in ['full', 'tied', 'diag', 'spherical']:
        g = mixture.GMM(n_components=n_components, covariance_type=cv_type,
                        random_state=rng, min_covar=1e-7)
        g.fit(X)
        aic = 2 * n_samples * SGH * n_dim + 2 * g._n_parameters()
        bic = (2 * n_samples * SGH * n_dim +
               np.log(n_samples) * g._n_parameters())
        bound = n_dim * 3. / np.sqrt(n_samples)
        assert np.abs(g.aic(X) - aic) / n_samples < bound
        assert np.abs(g.bic(X) - bic) / n_samples < bound

if __name__ == '__main__':
    import nose
    nose.runmodule()


import pandas as pd
import glob
import time
import numpy as num

inter=sorted(glob.glob('*****.csv'))

w='*****.xlsx'
table1=pd.read_excel(w, '*****', index_col=None, na_values=['NA']).fillna(0)

w='*****.csv'
tab=pd.read_csv(w).fillna(0)

tab.is_copy = False
pd.options.mode.chained_assignment = None

t1=time.time()

for i in range(len(tab)):
	if tab["IBR"][i]=='9A' or tab["IBR"][i] == '9B' or tab["IBR"][i] == '09A' or tab["IBR"][i] == '09B':
		tab["IBR"][i]='9'
	if tab["IBR"][i]=='11A' or tab["IBR"][i] == '11B' or tab["IBR"][i]=='11C' or tab["IBR"][i] == '11D' or tab["IBR"][i]=='36B':
		tab["IBR"][i]='11'
	if tab["IBR"][i]=='36A' or tab["IBR"][i] == '36B':
		tab["IBR"][i]='36'
	if tab["IBR"][i]=='13A' or tab["IBR"][i] == '13B' or tab["IBR"][i] == '13C':
		tab["IBR"][i]='13'
	if tab["IBR"][i]=='23A' or tab["IBR"][i] == '23B' or tab["IBR"][i] == '23E' or tab["IBR"][i] == '23F' or tab["IBR"][i] == '23H':
		tab["IBR"][i]='23'
	if tab["IBR"][i]=='26A' or tab["IBR"][i] == '26B' or tab["IBR"][i] == '26C' or tab["IBR"][i] == '26D' or tab["IBR"][i] == '26E':
		tab["IBR"][i]='26'
	if tab["IBR"][i]=='35A' or tab["IBR"][i] == '35B':
		tab["IBR"][i]='35'
	if tab["IBR"][i]=='36A':
		tab["IBR"][i]='36'
	if tab["IBR"][i]=='39A' or tab["IBR"][i] == '39B' or tab["IBR"][i] == '39C' or tab["IBR"][i] == '39D':
		tab["IBR"][i]='39'
	if tab["IBR"][i]=='40A' or tab["IBR"][i] == '40B' or tab["IBR"][i] == '40C':
		tab["IBR"][i]='40'
	if tab["IBR"][i]=='64A' or tab["IBR"][i] == '64B':
		tab["IBR"][i]='64'
	if tab["IBR"][i]=='90A' or tab["IBR"][i] == '90B' or tab["IBR"][i] == '90C' or tab["IBR"][i] == '90H' or tab["IBR"][i] == '90F' or tab["IBR"][i] == '90G' or tab["IBR"][i]=='90J' or tab["IBR"][i]=='90Z':
		tab["IBR"][i]='90'

#convert to string for the join
for i in range(len(table1)):
	table1['IBR_code'][i]=str(table1['IBR_code'][i])

description=table1.set_index([ "IBR_code"])

t2=time.time()
print t2-t1

#index crime
tab["index"]=num.nan

for i in range(len(tab)): #convert to integer
	tab["index"][i]=tab.index[i]+1

#join
tab=tab.join(description, on=["IBR"], sort=True, rsuffix='_1', how='outer').fillna(0)
tab=tab[(tab["Reported_address"] != 0)].reset_index(drop=True).fillna(0)
tab["IBR_description"]=tab["crime_des12"]

t3=time.time()
print t3-t2

tab=tab[["Global_ID","Reported_address","Incident_date","Incident_time","Report_date","Report_time","Latitude","Longitude","IBR","IBR_description","Police_Department_Code","PD_description","State_Statute_Literal","State_Statute_Number","flag_geocode",'Fdir_n1','Edir_n1','strname_n1','strtype_n1','Enum_n1','Fdir_n2','Edir_n2','strname_n2','strtype_n2','Enum_n2','comname','mroad1','mratio1','wcorr1','wratio1','mroad2','mratio2','wcorr2','wratio2','match']]
tab=tab.replace("",num.nan)
tab=tab.replace("0",num.nan)
tab=tab.replace("00",num.nan)
tab=tab.replace(0,num.nan)
tab.to_csv('*****.csv',index=False)

for i in range(len(tab)):
	tab['Global_ID'][i]=str(tab['Global_ID'][i])
description=tab.set_index([ "Global_ID"])

name1=[i[i.find('inter'):i.rfind('C.csv')+1].replace('_matchgeo','') for i in inter]

for p, q in zip((inter), (name1)):
	table1=pd.read_csv(p)
	for i in range(len(table1)):
		tab['Global_ID'][i]=str(tab['Global_ID'][i])
	table1=table1.join(description, on=["Global_ID"], sort=True, rsuffix='_1', how='outer').fillna(0)
	table1=table1[(table1["Reported_address"] != 0)].reset_index(drop=True).fillna(0)
	table1["IBR_description"]=table1["IBR_description_1"]
	table1["IBR"]=table1["IBR_1"]
	table1=table1[["Global_ID","Reported_address","Incident_date","Incident_time","Report_date","Report_time","Latitude","Longitude","IBR","IBR_description","Police_Department_Code","PD_description","State_Statute_Literal","State_Statute_Number","flag_geocode",'Fdir_n1','Edir_n1','strname_n1','strtype_n1','Enum_n1','Fdir_n2','Edir_n2','strname_n2','strtype_n2','Enum_n2','comname','mroad1','mratio1','wcorr1','wratio1','mroad2','mratio2','wcorr2','wratio2','match']]
	table1.to_csv('*****.csv',index=False)