import os
import datetime
import sys
import time
import string
import random
import pandas as pd
import numpy as np
import gc

if(len(sys.argv) < 2):
    print('Usage: CSVTrainer.py train.csv validation.csv model.h5 log.txt')
    sys.exit(1)

trainingName = sys.argv[1]
validationName = sys.argv[2]
modelName = sys.argv[3]
logName = sys.argv[4]

from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import *
import keras.preprocessing.image as image
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint, CSVLogger

from keras.layers import Input, merge, Dropout, Dense, Flatten, Activation
from keras.layers.convolutional import MaxPooling2D, Convolution2D, AveragePooling2D
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam, SGD
from keras.models import Model, load_model
from keras import regularizers

from keras import backend as K
from keras.utils.data_utils import get_file

from sklearn.metrics import accuracy_score

from keras.applications import resnet50

def readCSV(fileList):

    namesDataFrame = pd.read_csv(fileList)

    flatten = lambda l: [item for sublist in l for item in sublist]
    labels = sorted(list(set(flatten([l.split(' ') for l in namesDataFrame['tags'].values]))))

    labelMap = {l: i for i, l in enumerate(labels)}

    numberOfLabels = len(labels)
    numberOfImages = len(namesDataFrame)

    fileNames = []
    y = np.zeros((numberOfImages, numberOfLabels), np.float32)
    for index in range(0, numberOfImages):
        inputImage = image.img_to_array(image.load_img(namesDataFrame.iloc[index][0]))
        fileNames.append(namesDataFrame.iloc[index][0])
        tags = namesDataFrame.iloc[index][1]
        for t in tags.split(' '):
            y[index, labelMap[t]] = 1.0
    return (fileNames, y, labelMap)

print('Loading images..........', end = '',flush = True)

(trainingFileNames, trainY, trainingLabelMap) = readCSV(trainingName)
(validationFileNames, validationY, validationLabelMap) = readCSV(validationName)

print('done.', flush = True)

if len(trainingLabelMap) != len(validationLabelMap):
    print("Label maps for training and validation are not equal")
    sys.exit(1)

numberOfTrainingImages = len(trainingFileNames)
numberOfValidationImages = len(validationFileNames)
numberOfChannels = 3
nx = 256
ny = 256
batchSize = 25
lossName = 'binary_crossentropy'
activationName = 'sigmoid'

resnetModel = resnet50.ResNet50(include_top=False, weights='imagenet', input_shape=(numberOfChannels, nx, ny))

print('The number of layers in the resnet model = %d' % (len(resnetModel.layers)))

bottleneckTrainingDataGenerator = ImageDataGenerator(rescale = 1.0/255.0)
bottleneckValidationDataGenerator = ImageDataGenerator(rescale = 1.0/255.0)
bottleneckTrainingGenerator = bottleneckTrainingDataGenerator.flow_from_filenames(trainingFileNames, target_size = (nx, ny), batch_size = batchSize, shuffle = False)
bottleneckValidationGenerator = bottleneckTrainingDataGenerator.flow_from_filenames(validationFileNames, target_size = (nx, ny), batch_size = batchSize, shuffle = False)
bottleneckTrainingFeatures = resnetModel.predict_generator(bottleneckTrainingGenerator, numberOfTrainingImages)
bottleneckValidationFeatures = resnetModel.predict_generator(bottleneckValidationGenerator, numberOfValidationImages)

newTop = Sequential()
newTop.add(Flatten(input_shape = bottleneckTrainingFeatures.shape[1:]))
newTop.add(Dense(512, activation='relu'))
newTop.add(Dropout(0.5))
newTop.add(Dense(len(trainingLabelMap), activation=activationName, name='predictions'))
newTop.compile(loss=lossName, optimizer=Adam(lr=1.0E-3))

print('Fitting predicted features...', flush = True)

newTop.fit(bottleneckTrainingFeatures, trainY, validation_data = (bottleneckValidationFeatures, validationY), verbose = 1, batch_size = batchSize, nb_epoch = 25)

print('Done.', flush = True)

finalModel = Model(input = resnetModel.input, output = newTop(resnetModel.output))
print('The number of layers in the final model = %d' % (len(finalModel.layers)))

for layer in finalModel.layers[:(len(resnetModel.layers) - 21)]:
    layer.trainable = False

finalModel.compile(loss=lossName,optimizer=SGD(lr=1e-4, momentum=0.9))
print(finalModel.summary())

# Could add vertical_flip = True
trainingDataGenerator = ImageDataGenerator(rescale = 1.0/255.0, rotation_range = 40, zoom_range = 0.15, horizontal_flip = True,
                                           width_shift_range = 0.1, height_shift_range = 0.1, shear_range = 0.1)
validationDataGenerator = ImageDataGenerator(rescale = 1.0/255.0)
trainingGenerator = trainingDataGenerator.flow_from_filenames(trainingFileNames, trainY, batch_size = batchSize, target_size = (nx, ny))
validationGenerator = validationDataGenerator.flow_from_filenames(validationFileNames, validationY, batch_size = batchSize, target_size = (nx, ny))

csvLogger = CSVLogger(logName, append=True)
checkPointer = ModelCheckpoint(filepath=modelName, verbose = 1, save_best_only = True)
finalModel.fit_generator(trainingGenerator, numberOfTrainingImages, 50, validation_data = validationGenerator,
                         nb_val_samples = numberOfValidationImages, callbacks = [checkPointer, csvLogger])


import numpy as np
from numpy.testing import dec

import statsmodels.api as sm
from statsmodels.graphics.gofplots import qqplot, qqline, ProbPlot
from scipy import stats

try:
    import matplotlib.pyplot as plt
    import matplotlib
    have_matplotlib = True
except ImportError:
    have_matplotlib = False

class BaseProbplotMixin(object):
    def base_setup(self):
        if have_matplotlib:
            self.fig, self.ax = plt.subplots()
        self.other_array = np.random.normal(size=self.prbplt.data.shape)
        self.other_prbplot = sm.ProbPlot(self.other_array)

    def teardown(self):
        if have_matplotlib:
            plt.close('all')

    @dec.skipif(not have_matplotlib)
    def test_qqplot(self):
        self.fig = self.prbplt.qqplot(ax=self.ax, line=self.line)

    @dec.skipif(not have_matplotlib)
    def test_ppplot(self):
        self.fig = self.prbplt.ppplot(ax=self.ax, line=self.line)

    @dec.skipif(not have_matplotlib)
    def test_probplot(self):
        self.fig = self.prbplt.probplot(ax=self.ax, line=self.line)

    @dec.skipif(not have_matplotlib)
    def test_qqplot_other_array(self):
        self.fig = self.prbplt.qqplot(ax=self.ax, line=self.line,
                                        other=self.other_array)
    @dec.skipif(not have_matplotlib)
    def test_ppplot_other_array(self):
        self.fig = self.prbplt.ppplot(ax=self.ax, line=self.line,
                                        other=self.other_array)
    @dec.skipif(not have_matplotlib)
    def t_est_probplot_other_array(self):
        self.fig = self.prbplt.probplot(ax=self.ax, line=self.line,
                                        other=self.other_array)

    @dec.skipif(not have_matplotlib)
    def test_qqplot_other_prbplt(self):
        self.fig = self.prbplt.qqplot(ax=self.ax, line=self.line,
                                        other=self.other_prbplot)
    @dec.skipif(not have_matplotlib)
    def test_ppplot_other_prbplt(self):
        self.fig = self.prbplt.ppplot(ax=self.ax, line=self.line,
                                        other=self.other_prbplot)
    @dec.skipif(not have_matplotlib)
    def t_est_probplot_other_prbplt(self):
        self.fig = self.prbplt.probplot(ax=self.ax, line=self.line,
                                        other=self.other_prbplot)

    @dec.skipif(not have_matplotlib)
    def test_qqplot_custom_labels(self):
        self.fig = self.prbplt.qqplot(ax=self.ax, line=self.line,
                                      xlabel='Custom X-Label',
                                      ylabel='Custom Y-Label')

    @dec.skipif(not have_matplotlib)
    def test_ppplot_custom_labels(self):
        self.fig = self.prbplt.ppplot(ax=self.ax, line=self.line,
                                      xlabel='Custom X-Label',
                                      ylabel='Custom Y-Label')

    @dec.skipif(not have_matplotlib)
    def test_probplot_custom_labels(self):
        self.fig = self.prbplt.probplot(ax=self.ax, line=self.line,
                                        xlabel='Custom X-Label',
                                        ylabel='Custom Y-Label')

    @dec.skipif(not have_matplotlib)
    def test_qqplot_pltkwargs(self):
        self.fig = self.prbplt.qqplot(ax=self.ax, line=self.line,
                                      marker='d',
                                      markerfacecolor='cornflowerblue',
                                      markeredgecolor='white',
                                      alpha=0.5)

    @dec.skipif(not have_matplotlib)
    def test_ppplot_pltkwargs(self):
        self.fig = self.prbplt.ppplot(ax=self.ax, line=self.line,
                                      marker='d',
                                      markerfacecolor='cornflowerblue',
                                      markeredgecolor='white',
                                      alpha=0.5)

    @dec.skipif(not have_matplotlib)
    def test_probplot_pltkwargs(self):
        self.fig = self.prbplt.probplot(ax=self.ax, line=self.line,
                                        marker='d',
                                        markerfacecolor='cornflowerblue',
                                        markeredgecolor='white',
                                        alpha=0.5)

class TestProbPlotLongely(BaseProbplotMixin):
    def setup(self):
        np.random.seed(5)
        self.data = sm.datasets.longley.load()
        self.data.exog = sm.add_constant(self.data.exog, prepend=False)
        self.mod_fit = sm.OLS(self.data.endog, self.data.exog).fit()
        self.prbplt = sm.ProbPlot(self.mod_fit.resid, stats.t, distargs=(4,))
        self.line = 'r'
        self.base_setup()

class TestProbPlotRandomNormalMinimal(BaseProbplotMixin):
    def setup(self):
        np.random.seed(5)
        self.data = np.random.normal(loc=8.25, scale=3.25, size=37)
        self.prbplt = sm.ProbPlot(self.data)
        self.line = None
        self.base_setup()

class TestProbPlotRandomNormalWithFit(BaseProbplotMixin):
    def setup(self):
        np.random.seed(5)
        self.data = np.random.normal(loc=8.25, scale=3.25, size=37)
        self.prbplt = sm.ProbPlot(self.data, fit=True)
        self.line = 'q'
        self.base_setup()

class TestProbPlotRandomNormalLocScale(BaseProbplotMixin):
    def setup(self):
        np.random.seed(5)
        self.data = np.random.normal(loc=8.25, scale=3.25, size=37)
        self.prbplt = sm.ProbPlot(self.data, loc=8.25, scale=3.25)
        self.line = '45'
        self.base_setup()

class TestTopLevel(object):
    def setup(self):
        self.data = sm.datasets.longley.load()
        self.data.exog = sm.add_constant(self.data.exog, prepend=False)
        self.mod_fit = sm.OLS(self.data.endog, self.data.exog).fit()
        self.res = self.mod_fit.resid
        self.prbplt = sm.ProbPlot(self.mod_fit.resid, stats.t, distargs=(4,))
        self.other_array = np.random.normal(size=self.prbplt.data.shape)
        self.other_prbplot = sm.ProbPlot(self.other_array)

    def teardown(self):
        if have_matplotlib:
            plt.close('all')

    @dec.skipif(not have_matplotlib)
    def test_qqplot(self):
        fig = sm.qqplot(self.res, line='r')

    @dec.skipif(not have_matplotlib)
    def test_qqplot_2samples_ProbPlotObjects(self):
        # also tests all values for line
        for line in ['r', 'q', '45', 's']:
            # test with `ProbPlot` instances
            fig = sm.qqplot_2samples(self.prbplt, self.other_prbplot,
                                     line=line)
    @dec.skipif(not have_matplotlib)
    def test_qqplot_2samples_arrays(self):
        # also tests all values for line
        for line in ['r', 'q', '45', 's']:
            # test with arrays
            fig = sm.qqplot_2samples(self.res, self.other_array, line=line)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from sklearn.cross_validation import train_test_split
from sklearn import linear_model
from sklearn import mixture
from sklearn import metrics

import logging
import sys, traceback, os
import uuid
import psutil
import getpass

import usefulLibraryFiles         # Libreria propia del APP
import usefulLibrary             # Libreria propia del APP
import numpy as np

from datetime import datetime

log = logging.getLogger()
log.setLevel('INFO')

def applyRegression(dicWrongValues,L2,dicL,L_train,coefVaration,dicAlarmsRegr,score,testSize,analysis,procID):
    # Esta función tiene como objetivo:
    #   1º. Sustituir los "wrong" values de la lista L2 por valores predecidos según el resto de características.
    #       NOTA: cuidado cuando existan mas de una característica "wrong" para la misma muestra.
    #   2º. Validar uno a uno cada valor de la lista L2, es decir, comprobar que el valor real es el "mismo" que obtenemos al predecirlo. En
    #       el caso de que no sea (sea lo suficientemente diferente) generará una alarma.
    #
    #  Lo idóneo es unir el array sacado de la BD con el del data set a analizar (quitando las muestras cuyas coordenadas tienen valore raro),
    #  y ya con esta lista dividir la en train y test, esto es válido solo para el PASO 1
    #  NOTA: se podría definir una variable "regressionMinScore" para sólo aplicar la regresión en caso de que el score sea mayor a este valor.
    features=[]
    for col in sorted(dicL.keys()):
        features.append(dicL[col])
    feature_names = np.array(features)
    log.info(procID+" <applyRegression> Features: "+str(feature_names))
    if (len(dicWrongValues.keys())>0):
        L_full = usefulLibrary.unionListsWrong(dicWrongValues,L2,L_train,procID) # L_full contiene un array de la unión del array de
        # entrenamiento mas el del fichero sin los valores a 0 que tenían strings
        log.info(procID+" <applyRegression> Num columns L_full: "+str(len(L_full[0])))
        # Toca recorrer el array: PASO 1 --> en busca de los "0" que hemos puesto en los valores malos
        #   - la forma de recorrer esta vez no será registro a registro, lo haremos por columna ya que a la hora de aplicar el algoritmo
        # Sólo podremos aplicar la regresión lineal si se cumple que el tamaño del array L_full es:
        #       1. Mayor que el valor definido por la variable test_size
        #       2. Si es mayor lo divideremos hasta tener un array con el valor justo de "test_size", cogiendo muestras aleatorias
        percentSizeTrain=(len(L_full)*100)/(len(L_full)+len(dicWrongValues.keys()))
        if  int(percentSizeTrain) >= int(testSize):
            log.info(procID+" <applyRegression> STEP 1: Train array is upper to test_size "+str(testSize)+", the Lineal Regression will be executed.")
            analysis=True
            values_X, values_Y = [], []
            columns=[]
            for wrong in sorted(dicWrongValues.keys()):
                log.info(procID+" <applyRegression> WrongDict key= "+str(wrong))
                if dicWrongValues[wrong]["y"] not in columns:
                    columns.append(dicWrongValues[wrong]["y"])
            log.info(procID+" <applyRegression> Num columns of wrong values= "+str(len(columns)))
            for col in columns:
                log.info(procID+" <applyRegression> Col= "+str(col))
                values_Y, values_X = usefulLibrary.extractColumnArray(L_full,col,procID)
                log.info(procID+" <applyRegression> Num rows values_Y: "+str(len(values_Y)))
                log.info(procID+" <applyRegression> "+str(values_Y))
                log.info(procID+" <applyRegression> Num columns values_X: "+str(len(values_X[0])))
                values_X = np.array(values_X)
                values_Y = np.array(values_Y)
                # Antes de dividir tenemos que calcular el % del tamaño del test_size
                #
                perCsplit=(percentSizeTrain-int(testSize))/100 # EJ: 91% - 80% = 11% / 100 --> 0.11 del array a usar como test y 0.89 para train
                # haciendo justo el 80% (definido por la variable)
                log.info(procID+" <applyRegression> test_size= "+str(perCsplit))
                X_train, X_test, y_train, y_test =train_test_split(values_X, values_Y, test_size=perCsplit,random_state=33)
                log.debug(procID+" <applyRegression> X_train size: before= "+str(len(values_X))+" | now= "+str(len(X_train)))
                # Create linear regression object
                regr = linear_model.LinearRegression()
                # Train the model using the training sets
                regr.fit(X_train, y_train)
                # Explained variance score: 1 is perfect prediction
                log.info(procID+" <applyRegression> Variance score: "+str(regr.score(X_test,y_test)))
                # Ahora tocaría estimar y sustituir
                for reg in dicWrongValues.keys():
                    x=dicWrongValues[reg]["x"]
                    y=dicWrongValues[reg]["y"]
                    if L2[x][y] == 0 and y == col:  # nosotros le pusimos este valor, así nos aseguramosxº
                        y_pred=regr.predict(usefulLibrary.extractColumnList(L2[x],y,procID))
                        log.info(procID+" <applyRegression> Value predict for wrong value in coordinates "+str(x)+","+str(y)+": "+str(y_pred))
                        aprox=round(y_pred,4)
                        log.info(procID+" <applyRegression> Aproximate predict value: "+str(aprox))
                        # Ahora deberíamos sustituirlo en la Lista definitiva
                        L2[x][y]=aprox
        else:
            log.info(procID+" <applyRegression> STEP1: Train array is lower to test_size "+str(testSize)+", the Lineal Regression will not be executed.")
    # Para el PASO 2 no podemos unir la lista sacada de la BD con la del fichero ya que vamos a ir prediciendo cada valore del array del
    # fichero y no podemos usar los valores que ya vienen en el.
    log.info(procID+" <applyRegression> Num columns L_train: "+str(len(L_train[0])))
    percentSizeTrain=(len(L_train)*100)/(len(L_train)+len(L2))
    if  int(percentSizeTrain) >= int(testSize):
        log.info(procID+" <applyRegression> STEP 2: Train array is upper to test_size "+str(testSize)+", the Lineal Regression will be executed.")
        analysis=True
        values_X, values_Y = [], []
        # Nos toca recorre todo el array del fichero prediciendo uno a uno cada valor, por columna
        for colum in range(len(feature_names)):
            log.info(procID+" <applyRegression> Predict values of Colum= "+str(colum))
            values_Y, values_X = usefulLibrary.extractColumnArray(L_train,colum,procID)
            values_X = np.array(values_X)
            values_Y = np.array(values_Y)
            # Antes de dividir tenemos que calcular el % del tamaño del test_size
            #
            perCsplit=(percentSizeTrain-int(testSize))/100 # EJ: 91% - 80% = 11% / 100 --> 0.11 del array a usar como test y 0.89 para train
            # haciendo justo el 80% (definido por la variable)
            log.info(procID+" <applyRegression> test_size= "+str(perCsplit))
            X_train, X_test, y_train, y_test =train_test_split(values_X, values_Y, test_size=perCsplit,random_state=33)
            log.debug(procID+" <applyRegression> X_train size: before= "+str(len(values_X))+" | now= "+str(len(X_train)))
            # Create linear regression object
            regr = linear_model.LinearRegression()
            # Train the model using the training sets
            regr.fit(X_train, y_train)
            # Explained variance score: 1 is perfect prediction
            score=regr.score(X_test,y_test)
            log.info(procID+" - Variance score: "+str(score))
            # Una vez ya tenemos el estimador entrenado comenzamos a predecir
            for row in range(len(L2)):
                subDalarm={}
                newL = usefulLibrary.extractColumnList(L2[row],colum,procID)
                log.info(procID+" <applyRegression> List of features to predict: "+str(newL))
                y_pred=regr.predict(newL)
                log.info(procID+" <applyRegression> Value predict for coordinates row,colum "+str(row)+","+str(colum)+" -> REAL: "+str(L2[row][colum])+" PRED: "+str(y_pred))
                aprox=round(y_pred,4)
                log.info(procID+" <applyRegression> Aproximate predict value: "+str(aprox))
                coefV = usefulLibrary.desviacionTipica(L2[row][colum],aprox,procID)
                if coefV > int(coefVaration):
                    # Como el coeficiente de variación es mayor a lo permitido por variable generamos la alarma como posible anomalía
                    subDalarm["x"]=row
                    subDalarm["y"]=colum
                    dicAlarmsRegr[len(dicAlarmsRegr.keys())]=subDalarm
                    log.info(procID+" <applyRegression> Alarm generated...[coefV= "+str(coefV)+"]")
                else:
                    # Como el coeficiente de variación es menor a lo permitido por variable, consideramos que tanto el valor real como el predecido
                    # son semejantes por lo que no generaremos alarma
                    log.info(procID+" <applyRegression> Element with value between interval...[coefV= "+str(coefV)+"]")
    else:
        log.info(procID+" <applyRegression> STEP2: Train array is lower to test_size "+str(testSize)+", the Lineal Regression will not be executed.")
    # Una vez recorrido el array y obtenidas todas las posibles alarmas, hemos terminado.

def applyClusteringTotal(L_predict,features,L_train,dicDBvariables,dicAlarmsClus,score,procID):
    # EL objetivo de este procedimiento es aplicar un número de veces (según la variable proofGroup) el algoritmo de clustering
    # Gaussian Mixture Models (GMM) aumentanto en cada una de las iteraciones el número de grupos a obtener (2^x). En cada iteración se obtendrá
    # el grupo al que pertence cada una de las muestras del array a predecir (L_predict)
    # Se aplicará por muestra (row)
    log.info(procID+" <applyClusteringTotal> Features: "+str(features))
    percentSizeTrain=(len(L_train)*100)/(len(L_train)+len(L_predict))
    # Antes de dividir tenemos que calcular el % del tamaño del test_size
    #
    perCsplit=(percentSizeTrain-int(dicDBvariables["test_size"]))/100 # EJ: 91% - 80% = 11% / 100 --> 0.11 del array a usar como test y 0.89 para train
    # haciendo justo el 80% (definido por la variable)
    log.debug(procID+" <applyClusteringTotal> test_size= "+str(perCsplit))
    X_train, X_test, y_train, y_test =train_test_split(L_train, L_train, test_size=perCsplit,random_state=33)
    log.debug(procID+" <applyClusteringTotal> X_train size: before= "+str(len(L_train))+" | now= "+str(len(X_train)))
    nComp=2
    dicResultClusSample={}
    dicResultClusGroup={}
    for proof in range(int(dicDBvariables['proofGroup'])):
        log.info(procID+" <applyClusteringTotal> Proof level:"+str(proof)+" - n_components: "+str(nComp))
        gm = mixture.GMM(n_components=nComp,covariance_type='tied', random_state=42)
        gm.fit(X_train)
        y_pred = gm.predict(L_predict)
        usefulLibrary.saveResult(y_pred,dicResultClusSample,dicResultClusGroup,'I'+str(proof),procID)
        nComp=nComp*2
    log.debug(dicResultClusSample)
    log.debug(dicResultClusGroup)
    usefulLibrary.applyClusteringAlarm(dicResultClusSample,dicResultClusGroup,dicAlarmsClus,dicDBvariables['clustGroup'],procID)
    for alarm in sorted(dicAlarmsClus.keys()):
        log.info(procID+" <applyClusteringTotal> Row:"+str(L_predict[alarm])+" - level: "+str(dicAlarmsClus[alarm]))

def applyClusteringPartial(L_predict,features,L_train,dicDBvariables,dicAlarmsClusTotal,score,procID):
    # EL objetivo de este procedimiento es aplicar un número de veces (según la variable proofGroup) el algoritmo de clustering
    # Gaussian Mixture Models (GMM) aumentanto en cada una de las iteraciones el número de grupos a obtener (2^x). En cada iteración se obtendrá
    # el grupo al que pertence cada una de las muestras del array a predecir (L_predict)
    # Se aplicará por columna (column)
    log.info(procID+" <applyClusteringPartial> Features: "+str(features))
    percentSizeTrain=(len(L_train)*100)/(len(L_train)+len(L_predict))
    # Antes de dividir tenemos que calcular el % del tamaño del test_size
    #
    perCsplit=(percentSizeTrain-int(dicDBvariables["test_size"]))/100 # EJ: 91% - 80% = 11% / 100 --> 0.11 del array a usar como test y 0.89 para train
    # haciendo justo el 80% (definido por la variable)
    log.debug(procID+" <applyClusteringPartial> test_size= "+str(perCsplit))
    X_train, X_test, y_train, y_test =train_test_split(L_train, L_train, test_size=perCsplit,random_state=33)
    log.debug(procID+" <applyClusteringPartial> X_train size: before= "+str(len(L_train))+" | now= "+str(len(X_train)))
    for col in range(len(features)):
        dicAlarmsClus={}
        nComp=2
        dicResultClusSample={}
        dicResultClusGroup={}
        L_1colTR, L_restColTR = usefulLibrary.extractColumnArray(L_train,col,procID)
        L_1colPR, L_restColPR = usefulLibrary.extractColumnArray(L_predict,col,procID)
        for proof in range(int(dicDBvariables['proofGroup'])):
            log.info(procID+" <applyClusteringPartial> Proof level:"+str(proof)+" - n_components: "+str(nComp)+" - COLUMN= "+str(col))
            gm = mixture.GMM(n_components=nComp,covariance_type='tied', random_state=42)
            gm.fit(L_1colTR)
            y_pred = gm.predict(L_1colPR)
            usefulLibrary.saveResult(y_pred,dicResultClusSample,dicResultClusGroup,'I'+str(proof),procID)
            nComp=nComp*2
        log.debug(dicResultClusSample)
        log.debug(dicResultClusGroup)
        usefulLibrary.applyClusteringAlarm(dicResultClusSample,dicResultClusGroup,dicAlarmsClus,dicDBvariables['clustGroup'],procID)
        dicAlarmsClusTotal[col]=dicAlarmsClus
        for alarm in sorted(dicAlarmsClus.keys()):
            log.info(procID+" <applyClusteringPartial> COLUMN= "+str(col)+" Value:"+str(L_predict[alarm][col])+" - level: "+str(dicAlarmsClus[alarm]))


#!/usr/bin/env ipython
from pylab import *
import numpy as np
import console_colors as ccl
from scipy.io.netcdf import netcdf_file
import os, sys
import matplotlib.patches as patches
import matplotlib.transforms as transforms
from numpy import array
from matplotlib.gridspec import GridSpec
import matplotlib.pyplot as plt

class gral:
    def __init__(self):
        self.name='name'

TS  = 11
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

def makefig(ax, mc, sh, TEXT, TEXT_LOC, YLIMS, varname):
    LW  = 0.3                # linewidth
    MS  = 1.5
    fmc,fsh = 3.0, 1.0      # escaleos temporales
    if(varname == 'Temp'):
        mc.med      /= 1.0e4; sh.med      /= 1.0e4
        mc.avr      /= 1.0e4; sh.avr      /= 1.0e4
        mc.std_err  /= 1.0e4; sh.std_err  /= 1.0e4
        YLIMS[0]    /= 1.0e4; YLIMS[1] /= 1.0e4
        TEXT_LOC['mc'][1] /= 1.0e4
        TEXT_LOC['sh'][1] /= 1.0e4

    # curvas del mc
    time = fsh+fmc*mc.tnorm
    cc = time>=fsh
    ax.plot(time[cc], mc.avr[cc], 'o-', color='black', markersize=MS, label='mean', lw=LW)
    ax.plot(time[cc], mc.med[cc], 'o-', color='red', alpha=.8, markersize=MS, markeredgecolor='none', label='median', lw=LW)
    # sombra del mc
    inf     = mc.avr + mc.std_err/np.sqrt(mc.nValues)
    sup     = mc.avr - mc.std_err/np.sqrt(mc.nValues)
    ax.fill_between(time[cc], inf[cc], sup[cc], facecolor='gray', alpha=0.5)
    trans   = transforms.blended_transform_factory(
                ax.transData, ax.transAxes)
    rect1   = patches.Rectangle((fsh, 0.), width=fmc, height=1,
                transform=trans, color='blue',
                alpha=0.3)
    ax.add_patch(rect1)

    # curvas del sheath
    time = fsh*sh.tnorm
    cc = time<=fsh
    ax.plot(time[cc], sh.avr[cc], 'o-', color='black', markersize=MS, lw=LW)
    ax.plot(time[cc], sh.med[cc], 'o-', color='red', alpha=.8, markersize=MS, markeredgecolor='none', lw=LW)
    # sombra del sheath
    inf     = sh.avr + sh.std_err/np.sqrt(sh.nValues)
    sup     = sh.avr - sh.std_err/np.sqrt(sh.nValues)
    ax.fill_between(time[cc], inf[cc], sup[cc], facecolor='gray', alpha=0.5)
    #trans   = transforms.blended_transform_factory(
    #            ax.transData, ax.transAxes)
    rect1   = patches.Rectangle((0., 0.), width=fsh, height=1,
                transform=trans, color='orange',
                alpha=0.3)
    ax.add_patch(rect1)

    #ax.legend(loc='best', fontsize=10)
    ax.tick_params(labelsize=TS)
    ax.grid()
    ax.set_xlim(-2.0, 7.0)
    ax.set_ylim(YLIMS)
    ax.text(TEXT_LOC['mc'][0], TEXT_LOC['mc'][1], TEXT['mc'], fontsize=7)
    ax.text(TEXT_LOC['sh'][0], TEXT_LOC['sh'][1], TEXT['sh'], fontsize=7)

    if(varname in ('beta','Temp', 'rmsB', 'rmsBoB')):
        ax.set_yscale('log')
    else:
        ax.set_yscale('linear')

    return ax

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
stf = {}
stf['B']    = {
                'label': 'B [nT]',
                'ylims': [5., 29.],
                'text_loc_1': {'mc':[4.5, 15.0], 'sh':[-1.95, 12.0]},
                'text_loc_2': {'mc':[4.5, 18.0], 'sh':[-1.95, 12.0]},
                'text_loc_3': {'mc':[4.5, 12.0], 'sh':[-1.95, 12.0]},
                'nrow': 1
                }
stf['V']    = {
                'label': 'Vsw [km/s]',
                'ylims': [350., 800.],
                'text_loc_1': {'mc':[4.5, 500.0], 'sh':[-1.95, 520.0]},
                'text_loc_2': {'mc':[4.5, 600.0], 'sh':[-1.95, 600.0]},
                'text_loc_3': {'mc':[4.5, 410.0], 'sh':[-1.95, 600.0]},
                'nrow': 2
                }
stf['rmsBoB']    = {
                'label': 'rmsBoB [1]',
                'ylims': [0.015, 0.21],
                'text_loc_1': {'mc':[4.5, 0.020], 'sh':[-1.95, 0.02]},
                'text_loc_2': {'mc':[4.5, 0.095], 'sh':[-1.95, 0.02]},
                'text_loc_3': {'mc':[4.5, 0.099], 'sh':[-1.95, 0.02]},
                'nrow': 6
                }
stf['rmsB']    = {
                'label': 'rmsB [nT]',
                'ylims': [0.1, 4.0],
                'text_loc_1': {'mc':[4.5, 1.0], 'sh':[-1.95, 1.3]},
                'text_loc_2': {'mc':[4.5, 1.0], 'sh':[-1.95, 1.3]},
                'text_loc_3': {'mc':[4.5, 1.0], 'sh':[-1.95, 1.3]},
                'nrow': 1
                }
stf['beta']    = {
                'label': '$\\beta$ [1]',
                'ylims': [0.02, 10.0],
                'text_loc_1': {'mc':[4.5, 0.1], 'sh':[-1.95, 0.2]},
                'text_loc_2': {'mc':[4.5, 0.1], 'sh':[-1.95, 0.2]},
                'text_loc_3': {'mc':[4.5, 0.1], 'sh':[-1.95, 0.2]},
                'nrow': 5
                }
stf['Pcc']    = {
                'label': '$n_p$ [$cm^{-3}$]',
                'ylims': [1, 23],
                'text_loc_1': {'mc':[4.5, 14], 'sh':[-1.95, 16.0]},
                'text_loc_2': {'mc':[4.5, 14], 'sh':[-1.95, 16.0]},
                'text_loc_3': {'mc':[4.5, 11], 'sh':[-1.95, 18.0]},
                'nrow': 3
                }
stf['Temp']    = {
                'label': 'T ($\\times 10^4$) [K]',
                'ylims': [1e4, 100e4],
                'text_loc_1': {'mc':[4.5, 18.0e4], 'sh':[-1.95, 20.0e4]},
                'text_loc_2': {'mc':[4.5,  2.0e4], 'sh':[-1.95, 20.0e4]},
                'text_loc_3': {'mc':[4.5,  2.0e4], 'sh':[-1.95, 20.0e4]},
                'nrow': 4
                }
stf['AlphaRatio']    = {
                'label': 'alpha ratio [1]',
                'ylims': [0.02, 0.09],
                'text_loc_1': {'mc':[4.5, 0.022], 'sh':[-1.95, 0.07]},
                'text_loc_2': {'mc':[4.5, 0.022], 'sh':[-1.95, 0.07]},
                'text_loc_3': {'mc':[4.5, 0.022], 'sh':[-1.95, 0.07]}
                }
stf['CRs']    = {
                'label': '$n_{GCR}$ [%]',
                'ylims': [-8.0, 2.0],
                'text_loc_1': {'mc':[4.5, -4.0], 'sh':[-1.95, -4.5]},
                'text_loc_2': {'mc':[4.5, -7.0], 'sh':[-1.95, -4.5]},
                'text_loc_3': {'mc':[4.5, -7.5], 'sh':[-1.95, -4.5]},
                'nrow': 2
                }

TEXT = {}

dir_figs        = sys.argv[1] #'../figs'
#dir_inp_mc      = '../../../../mcs/ascii/MCflag2/wShiftCorr/_test_Vmc_'
#dir_inp_sh      = '../../../../sheaths/ascii/MCflag2/wShiftCorr/_test_Vmc_'
dir_inp_mc      = os.environ['RIGHT']
dir_inp_sh      = os.environ['LEFT']
vlo     = [100.0, 450.0, 550.0]
vhi     = [450.0, 550.0, 3000.0]
nvars   = len(stf.keys())

print " input: "
print " %s " % dir_inp_mc
print " %s \n" % dir_inp_sh
print " vlo, vhi: ", (vlo, vhi), '\n'
print " nvars: ", nvars
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

i=2
#fig = figure(1, figsize=(12, 15))
f   = plt.figure(1, figsize=(7, 5.8))
nr  = 1         # scale for row size
gs  = GridSpec(nrows=3*nr, ncols=2*3)
gs.update(left=0.1, right=0.98, hspace=0.13, wspace=0.15)

for i in range(3):
    fname_inp = 'MCflag2_2before.4after_fgap0.2_Wang90.0_vlo.%3.1f.vhi.%3.1f' % (vlo[i], vhi[i])
    fname_inp_nro_mc   = dir_inp_mc + '/n.events_' + fname_inp + '.txt'
    fname_inp_nro_sh   = dir_inp_sh + '/n.events_' + fname_inp + '.txt'

    #n       = 1     # number of row
    print " ______ col %d ______" % i

    for varname in ('rmsB', 'CRs'):
        # abro el file para averiguar el nro de eventos
        fnro_mc = open(fname_inp_nro_mc, 'r')
        fnro_sh = open(fname_inp_nro_sh, 'r')
        for lmc, lsh in zip(fnro_mc, fnro_sh):
            l_mc    = lmc.split()
            l_sh    = lsh.split()
            if varname==l_mc[0]:       # nombre de la variable
                n       = stf[varname]['nrow']
                ax      = plt.subplot(gs[(n-1)*nr:n*nr, (2*i):(2*(i+1))])
                Nfinal_mc, Nfinal_sh  = int(l_mc[1]), int(l_sh[1]) # nmbr of events
                fnro_mc.close(); fnro_sh.close()
                break

        print " %s"%varname, '  Nfinal_mc:%d' % Nfinal_mc, 'Nfinal_sh:%d' % Nfinal_sh
        mc, sh  = gral(), gral()

        fname_inp_mc = dir_inp_mc + '/' + fname_inp + '_%s.txt' % varname
        fname_inp_sh = dir_inp_sh + '/' + fname_inp + '_%s.txt' % varname
        mc.tnorm, mc.med, mc.avr, mc.std_err, mc.nValues = np.loadtxt(fname_inp_mc).T
        sh.tnorm, sh.med, sh.avr, sh.std_err, sh.nValues = np.loadtxt(fname_inp_sh).T

        # nro de datos con mas del 80% non-gap data
        TEXT['mc']  = ' N: %d'  % Nfinal_mc
        TEXT['sh']  = ' N: %d'  % Nfinal_sh
        if(vlo[i]==100.0):
            TEXT_LOC    = stf[varname]['text_loc_1'] #1.7, 12.0
        elif(vlo[i]==450.0):
            TEXT_LOC    = stf[varname]['text_loc_2'] #1.7, 12.0
        elif(vlo[i]==550.0):
            TEXT_LOC    = stf[varname]['text_loc_3'] #1.7, 12.0
        else:
            print " ----> ERROR con 'v_lo'!"
            raise SystemExit

        ylims       = array(stf[varname]['ylims']) #[4., 17.]
        ylabel      = stf[varname]['label'] #'B [nT]'
        ax = makefig(ax, mc, sh, TEXT, TEXT_LOC, ylims, varname)

        # ticks & labels x
        ax.tick_params(labelsize=TS)
        if n==2: #n==nvars-1:
            ax.set_xlabel('time normalized to\nsheath/MC passage [1]', fontsize=11)
            #ax.xaxis.set_ticklabels([-1,0,1,2,3])
            xticks = [-2,-1,0,1,2,3,4,5,6,7]
            ax.set_xticks(xticks)
            ax.set_xticklabels(xticks)
        else:
            ax.set_xlabel('')
            #ax.get_xaxis().set_ticks([])
            ax.xaxis.set_ticklabels([])

        # ticks & labels y
        if i==0:
            ax.set_ylabel(ylabel, fontsize=15)
        else:
            ax.set_ylabel('')
            #ax.get_yaxis().set_ticks([])
            ax.yaxis.set_ticklabels([])

#+++++++++++++++++++++++++ nCR & model-fit
#dirs            = {}
#dirs['sheath']  = '../../../../sheaths/ascii/MCflag2/wShiftCorr/_test_Vmc_'
#dirs['mc']      = '../../../../mcs/ascii/MCflag2/wShiftCorr/_test_Vmc_'
#dirs['fname_inputs'] = 'MCflag2_2before.4after_fgap0.2_Wang90.0'
#dirs['figs']    = dir_figs
#
#par = {}
#par['lo']   = {
#    'vlo': 100.0,
#    'vhi': 450.0,
#    'tau': 2.36,
#    'bp' : 0.0,
#    'q'  : -9.373,
#    'off': 0.89,
#    'bo' : 16.15
#}
#par['mid'] = {
#    'vlo': 450.0,
#    'vhi': 550.0,
#    'tau': 4.18,
#    'bp' : -0.9,
#    'q'  : -6.02,
#    'off': 0.0,
#    'bo' : 11.87
#}
#par['hi'] = {
#    'vlo': 550.0,
#    'vhi': 3000.0,
#    'tau': 5.78,
#    'bp' : -0.18,
#    'q'  : -5.53,
#    'off': 1.01,
#    'bo' : 14.48
#}
#
#from funcs import build_plot
#n   = 3; i=0
#for i, name in zip(range(3), ('lo', 'mid', 'hi')):
#    ax  = plt.subplot(gs[(n-1)*nr:n*nr, (2*i):(2*(i+1))])
#    build_plot(dirs, par[name], ax)
#    if i==0:
#        ax.set_ylabel('$n_{GCR}$ [%]', fontsize=15)
#    else:
#        ax.set_ylabel('')
#        ax.yaxis.set_ticklabels([])

#+++++++++++++++++++++++++++++++++++++++++

#fig.tight_layout()
#fname_fig   = dir_figs + '/fig_vlo.%3.1f_vhi.%3.1f_%s.png'%(vlo, vhi, varname)
fname_fig = '%s/figs_splitted_3.png' % dir_figs
savefig(fname_fig, dpi=150, bbox_inches='tight')
close()

print "\n output en:\n %s\n" % fname_fig
#EOF


"""
Author: Chukwubuikem Ume-Ugwa
Email: chubiyke@gmail.com

MIT License

Copyright (c) 2017 CleverChuk

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from dataparser import *

from multiprocessing import Pool, Manager
import os
import time

manager = Manager()
heightD = manager.dict() # holds values for minimum height of each particle

TSIZE = 8 # data type size in bytes
N_OFFSETS = 44 # number of data
FCOLOR = genColor(N_OFFSETS, manager)

# Dimension of the simulation bed
xsize = 78
ysize = 112
zsize = 104

hOut = "HeightData"
def startChild(fname):
	# DISALLOWED IN PYTHON
	iam.fn = fname
	dictn = iam.manager.dict()
	mylist = iam.manager.list()

	pool = Pool()
	# passing offset multiplier to the producer task
	pool.map(iam.producer, [i for i in range(1 , iam.N_OFFSETS)], 1)

	# Feeds task from producers into the list
	for i, j in self.dictn.items():
		mylist.append(j[0])

	# single process to handle plotting
	proc = Process(target=iam.consumer, args=(mylist, ))
	proc.start()
	proc.join()

def multijob(fname):
	"""
	Handles reading and plotting of data in file with name fname
	"""

	print("Starting multijob from process: %d" % os.getpid())

	fig = plt.figure()
	axis = Axes3D(fig)
	heightL = manager.list()

	axis = Axes3D(fig)
	axis.set_xlim([0,ysize])
	axis.set_ylim([0,ysize])
	axis.set_zlim([0,ysize])
	axis.view_init(elev = 40, azim = 50)

	coords = manager.list()
	rho = readsingle(fname)

	for i in range(1, N_OFFSETS):

		eta_s = readsingle(fname, i * TSIZE)
#		eta_s = process(rho, filter_eta(eta_s))
		coords.append(getcoords(eta_s, xsize, ysize, zsize))
		heightL.append(max(coords[-1][-2]) - min(coords[-1][-2]))

	writtable(hOut,str(heightL).strip('[]'))

	plot(coords, fig, axis, count = "ALL", fcolor = FCOLOR, multijob = (True,fname))
	print("Finished multijob from process: %d" % os.getpid())

if __name__ == "__main__":
	print("Starting mutiple jobs in a process task")
	import timeit, sys
	start_time = timeit.default_timer()
	if(os.path.exists(hOut)):
		os.remove(hOut)

	pool = Pool()
	files = list()

	MAXCOUNT = 4
	STEP = 2
	START = 0
	FNAME = "fullT{0}.dat"

	## file with filesname to work on
	for i in range(START, MAXCOUNT, STEP):
		files.append(FNAME.format(i))

	pool.map(multijob, files, 1)
	elapsed = timeit.default_timer() - start_time

	print("total time %d seconds" % elapsed)
	print("Finished multiple job in a process task")


"""
======================================================================
Compressive sensing: tomography reconstruction with L1 prior (Lasso)
======================================================================

This example shows the reconstruction of an image from a set of parallel
projections, acquired along different angles. Such a dataset is acquired in
**computed tomography** (CT).

Without any prior information on the sample, the number of projections
required to reconstruct the image is of the order of the linear size
``l`` of the image (in pixels). For simplicity we consider here a sparse
image, where only pixels on the boundary of objects have a non-zero
value. Such data could correspond for example to a cellular material.
Note however that most images are sparse in a different basis, such as
the Haar wavelets. Only ``l/7`` projections are acquired, therefore it is
necessary to use prior information available on the sample (its
sparsity): this is an example of **compressive sensing**.

The tomography projection operation is a linear transformation. In
addition to the data-fidelity term corresponding to a linear regression,
we penalize the L1 norm of the image to account for its sparsity. The
resulting optimization problem is called the :ref:`lasso`. We use the
class :class:`sklearn.linear_model.Lasso`, that uses the coordinate descent
algorithm. Importantly, this implementation is more computationally efficient
on a sparse matrix, than the projection operator used here.

The reconstruction with L1 penalization gives a result with zero error
(all pixels are successfully labeled with 0 or 1), even if noise was
added to the projections. In comparison, an L2 penalization
(:class:`sklearn.linear_model.Ridge`) produces a large number of labeling
errors for the pixels. Important artifacts are observed on the
reconstructed image, contrary to the L1 penalization. Note in particular
the circular artifact separating the pixels in the corners, that have
contributed to fewer projections than the central disk.
"""

print(__doc__)

# Author: Emmanuelle Gouillart <emmanuelle.gouillart@nsup.org>
# License: BSD 3 clause

import numpy as np
from scipy import sparse
from scipy import ndimage
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
import matplotlib.pyplot as plt

def _weights(x, dx=1, orig=0):
    x = np.ravel(x)
    floor_x = np.floor((x - orig) / dx)
    alpha = (x - orig - floor_x * dx) / dx
    return np.hstack((floor_x, floor_x + 1)), np.hstack((1 - alpha, alpha))

def _generate_center_coordinates(l_x):
    X, Y = np.mgrid[:l_x, :l_x]
    center = l_x / 2.
    X += 0.5 - center
    Y += 0.5 - center
    return X, Y

def build_projection_operator(l_x, n_dir):
    """ Compute the tomography design matrix.

    Parameters
    ----------

    l_x : int
        linear size of image array

    n_dir : int
        number of angles at which projections are acquired.

    Returns
    -------
    p : sparse matrix of shape (n_dir l_x, l_x**2)
    """
    X, Y = _generate_center_coordinates(l_x)
    angles = np.linspace(0, np.pi, n_dir, endpoint=False)
    data_inds, weights, camera_inds = [], [], []
    data_unravel_indices = np.arange(l_x ** 2)
    data_unravel_indices = np.hstack((data_unravel_indices,
                                      data_unravel_indices))
    for i, angle in enumerate(angles):
        Xrot = np.cos(angle) * X - np.sin(angle) * Y
        inds, w = _weights(Xrot, dx=1, orig=X.min())
        mask = np.logical_and(inds >= 0, inds < l_x)
        weights += list(w[mask])
        camera_inds += list(inds[mask] + i * l_x)
        data_inds += list(data_unravel_indices[mask])
    proj_operator = sparse.coo_matrix((weights, (camera_inds, data_inds)))
    return proj_operator

def generate_synthetic_data():
    """ Synthetic binary data """
    rs = np.random.RandomState(0)
    n_pts = 36.
    x, y = np.ogrid[0:l, 0:l]
    mask_outer = (x - l / 2) ** 2 + (y - l / 2) ** 2 < (l / 2) ** 2
    mask = np.zeros((l, l))
    points = l * rs.rand(2, n_pts)
    mask[(points[0]).astype(np.int), (points[1]).astype(np.int)] = 1
    mask = ndimage.gaussian_filter(mask, sigma=l / n_pts)
    res = np.logical_and(mask > mask.mean(), mask_outer)
    return res - ndimage.binary_erosion(res)

# Generate synthetic images, and projections
l = 128
proj_operator = build_projection_operator(l, l / 7.)
data = generate_synthetic_data()
proj = proj_operator * data.ravel()[:, np.newaxis]
proj += 0.15 * np.random.randn(*proj.shape)

# Reconstruction with L2 (Ridge) penalization
rgr_ridge = Ridge(alpha=0.2)
rgr_ridge.fit(proj_operator, proj.ravel())
rec_l2 = rgr_ridge.coef_.reshape(l, l)

# Reconstruction with L1 (Lasso) penalization
# the best value of alpha was determined using cross validation
# with LassoCV
rgr_lasso = Lasso(alpha=0.001)
rgr_lasso.fit(proj_operator, proj.ravel())
rec_l1 = rgr_lasso.coef_.reshape(l, l)

plt.figure(figsize=(8, 3.3))
plt.subplot(131)
plt.imshow(data, cmap=plt.cm.gray, interpolation='nearest')
plt.axis('off')
plt.title('original image')
plt.subplot(132)
plt.imshow(rec_l2, cmap=plt.cm.gray, interpolation='nearest')
plt.title('L2 penalization')
plt.axis('off')
plt.subplot(133)
plt.imshow(rec_l1, cmap=plt.cm.gray, interpolation='nearest')
plt.title('L1 penalization')
plt.axis('off')

plt.subplots_adjust(hspace=0.01, wspace=0.01, top=1, bottom=0, left=0,
                    right=1)

plt.show()


#!/usr/bin/env python
import os
import os.path as pt
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import argparse

#TODO: take decimal places as parameter for printing.
def sizeof_pp(num):
    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']:
        if abs(num) < 1024.0:
            return "%3.2f %s" % (num, unit)
        num /= 1024.0
    return "%.2f %s" % (num, 'Yi')

def xtic_formatter(num, tick_index):
    return sizeof_pp(num)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='.')
    parser.add_argument('dir_path', metavar='Path', type=str, help='')
    parser.add_argument('-p', '--plot', action='store_true')

    args = parser.parse_args()

    sizes = []
    symlink_count = 0
    for root, dirs, files in os.walk(args.dir_path, followlinks=False):
        for name in files:
            fullpath = pt.join(root, name)
            if not os.path.islink(fullpath):
                sizes.append(pt.getsize(fullpath))
            else:
                symlink_count += 1

    sizes.sort()
    print("Searching in directory: {0}".format(args.dir_path))
    print("Files Inspected: {0}".format(len(sizes)))
    print("Maxfilesize: " + sizeof_pp(sizes[-1]))
    print("Symlinks found: {0}".format(symlink_count))
    percentile = 95
    index = len(sizes) * (percentile / 100.)
    print("{0}% of files smaller than: ~".format(percentile) + sizeof_pp(
        sizes[int(index)]))

    sizesArray = np.asarray(sizes)
    if (args.plot):
        bins = min(len(sizes) / 10, 200)
        plt.figure(figsize=(8, 8))
        ax = plt.subplot(111)
        # Adjust y-axis to show bins of height 1 and max bin height.
        n, _, _ = plt.hist(sizesArray, bins, log=True)
        plt.ylim(0.5, max(n) * 1.1)
        plt.xlabel("File Size (bytes)")
        plt.ylabel("Log(Number of Files)")
        plt.title("File size histogram for: {0}".format(args.dir_path))
        x_formatter = mpl.ticker.ScalarFormatter(useOffset=False)
        x_formatter.set_scientific(False)
        x_format = mpl.ticker.FuncFormatter(xtic_formatter)
        ax.xaxis.set_major_formatter(x_format)
        plt.show()


"""Loader for the Labeled Faces in the Wild (LFW) dataset

This dataset is a collection of JPEG pictures of famous people collected
over the internet, all details are available on the official website:

    http://vis-www.cs.umass.edu/lfw/

Each picture is centered on a single face. The typical task is called
Face Verification: given a pair of two pictures, a binary classifier
must predict whether the two images are from the same person.

An alternative task, Face Recognition or Face Identification is:
given the picture of the face of an unknown person, identify the name
of the person by referring to a gallery of previously seen pictures of
identified persons.

Both Face Verification and Face Recognition are tasks that are typically
performed on the output of a model trained to perform Face Detection. The
most popular model for Face Detection is called Viola-Johns and is
implemented in the OpenCV library. The LFW faces were extracted by this face
detector from various online websites.
"""
# Copyright (c) 2011 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

from os import listdir, makedirs, remove
from os.path import join, exists, isdir

from sklearn.utils import deprecated

import logging
import numpy as np

try:
    import urllib.request as urllib  # for backwards compatibility
except ImportError:
    import urllib

from .base import get_data_home, Bunch
from ..externals.joblib import Memory

from ..externals.six import b

logger = logging.getLogger(__name__)

BASE_URL = "http://vis-www.cs.umass.edu/lfw/"
ARCHIVE_NAME = "lfw.tgz"
FUNNELED_ARCHIVE_NAME = "lfw-funneled.tgz"
TARGET_FILENAMES = [
    'pairsDevTrain.txt',
    'pairsDevTest.txt',
    'pairs.txt',
]

def scale_face(face):
    """Scale back to 0-1 range in case of normalization for plotting"""
    scaled = face - face.min()
    scaled /= scaled.max()
    return scaled

#
# Common private utilities for data fetching from the original LFW website
# local disk caching, and image decoding.
#

def check_fetch_lfw(data_home=None, funneled=True, download_if_missing=True):
    """Helper function to download any missing LFW data"""
    data_home = get_data_home(data_home=data_home)
    lfw_home = join(data_home, "lfw_home")

    if funneled:
        archive_path = join(lfw_home, FUNNELED_ARCHIVE_NAME)
        data_folder_path = join(lfw_home, "lfw_funneled")
        archive_url = BASE_URL + FUNNELED_ARCHIVE_NAME
    else:
        archive_path = join(lfw_home, ARCHIVE_NAME)
        data_folder_path = join(lfw_home, "lfw")
        archive_url = BASE_URL + ARCHIVE_NAME

    if not exists(lfw_home):
        makedirs(lfw_home)

    for target_filename in TARGET_FILENAMES:
        target_filepath = join(lfw_home, target_filename)
        if not exists(target_filepath):
            if download_if_missing:
                url = BASE_URL + target_filename
                logger.warning("Downloading LFW metadata: %s", url)
                urllib.urlretrieve(url, target_filepath)
            else:
                raise IOError("%s is missing" % target_filepath)

    if not exists(data_folder_path):

        if not exists(archive_path):
            if download_if_missing:
                logger.warning("Downloading LFW data (~200MB): %s", archive_url)
                urllib.urlretrieve(archive_url, archive_path)
            else:
                raise IOError("%s is missing" % target_filepath)

        import tarfile
        logger.info("Decompressing the data archive to %s", data_folder_path)
        tarfile.open(archive_path, "r:gz").extractall(path=lfw_home)
        remove(archive_path)

    return lfw_home, data_folder_path

def _load_imgs(file_paths, slice_, color, resize):
    """Internally used to load images"""

    # Try to import imread and imresize from PIL. We do this here to prevent
    # the whole sklearn.datasets module from depending on PIL.
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
        from scipy.misc import imresize
    except ImportError:
        raise ImportError("The Python Imaging Library (PIL)"
                          " is required to load data from jpeg files")

    # compute the portion of the images to load to respect the slice_ parameter
    # given by the caller
    default_slice = (slice(0, 250), slice(0, 250))
    if slice_ is None:
        slice_ = default_slice
    else:
        slice_ = tuple(s or ds for s, ds in zip(slice_, default_slice))

    h_slice, w_slice = slice_
    h = (h_slice.stop - h_slice.start) // (h_slice.step or 1)
    w = (w_slice.stop - w_slice.start) // (w_slice.step or 1)

    if resize is not None:
        resize = float(resize)
        h = int(resize * h)
        w = int(resize * w)

    # allocate some contiguous memory to host the decoded image slices
    n_faces = len(file_paths)
    if not color:
        faces = np.zeros((n_faces, h, w), dtype=np.float32)
    else:
        faces = np.zeros((n_faces, h, w, 3), dtype=np.float32)

    # iterate over the collected file path to load the jpeg files as numpy
    # arrays
    for i, file_path in enumerate(file_paths):
        if i % 1000 == 0:
            logger.info("Loading face #%05d / %05d", i + 1, n_faces)

        # Checks if jpeg reading worked. Refer to issue #3594 for more
        # details.
        img = imread(file_path)
        if img.ndim is 0:
            raise RuntimeError("Failed to read the image file %s, "
                               "Please make sure that libjpeg is installed"
                               % file_path)

        face = np.asarray(img[slice_], dtype=np.float32)
        face /= 255.0  # scale uint8 coded colors to the [0.0, 1.0] floats
        if resize is not None:
            face = imresize(face, resize)
        if not color:
            # average the color channels to compute a gray levels
            # representaion
            face = face.mean(axis=2)

        faces[i, ...] = face

    return faces

#
# Task #1:  Face Identification on picture with names
#

def _fetch_lfw_people(data_folder_path, slice_=None, color=False, resize=None,
                      min_faces_per_person=0):
    """Perform the actual data loading for the lfw people dataset

    This operation is meant to be cached by a joblib wrapper.
    """
    # scan the data folder content to retain people with more that
    # `min_faces_per_person` face pictures
    person_names, file_paths = [], []
    for person_name in sorted(listdir(data_folder_path)):
        folder_path = join(data_folder_path, person_name)
        if not isdir(folder_path):
            continue
        paths = [join(folder_path, f) for f in listdir(folder_path)]
        n_pictures = len(paths)
        if n_pictures >= min_faces_per_person:
            person_name = person_name.replace('_', ' ')
            person_names.extend([person_name] * n_pictures)
            file_paths.extend(paths)

    n_faces = len(file_paths)
    if n_faces == 0:
        raise ValueError("min_faces_per_person=%d is too restrictive" %
                         min_faces_per_person)

    target_names = np.unique(person_names)
    target = np.searchsorted(target_names, person_names)

    faces = _load_imgs(file_paths, slice_, color, resize)

    # shuffle the faces with a deterministic RNG scheme to avoid having
    # all faces of the same person in a row, as it would break some
    # cross validation and learning algorithms such as SGD and online
    # k-means that make an IID assumption

    indices = np.arange(n_faces)
    np.random.RandomState(42).shuffle(indices)
    faces, target = faces[indices], target[indices]
    return faces, target, target_names

def fetch_lfw_people(data_home=None, funneled=True, resize=0.5,
                     min_faces_per_person=0, color=False,
                     slice_=(slice(70, 195), slice(78, 172)),
                     download_if_missing=True):
    """Loader for the Labeled Faces in the Wild (LFW) people dataset

    This dataset is a collection of JPEG pictures of famous people
    collected on the internet, all details are available on the
    official website:

        http://vis-www.cs.umass.edu/lfw/

    Each picture is centered on a single face. Each pixel of each channel
    (color in RGB) is encoded by a float in range 0.0 - 1.0.

    The task is called Face Recognition (or Identification): given the
    picture of a face, find the name of the person given a training set
    (gallery).

    The original images are 250 x 250 pixels, but the default slice and resize
    arguments reduce them to 62 x 74.

    Parameters
    ----------
    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    funneled : boolean, optional, default: True
        Download and use the funneled variant of the dataset.

    resize : float, optional, default 0.5
        Ratio used to resize the each face picture.

    min_faces_per_person : int, optional, default None
        The extracted dataset will only retain pictures of people that have at
        least `min_faces_per_person` different pictures.

    color : boolean, optional, default False
        Keep the 3 RGB channels instead of averaging them to a single
        gray level channel. If color is True the shape of the data has
        one more dimension than than the shape with color = False.

    slice_ : optional
        Provide a custom 2D slice (height, width) to extract the
        'interesting' part of the jpeg files and avoid use statistical
        correlation from the background

    download_if_missing : optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    dataset : dict-like object with the following attributes:

    dataset.data : numpy array of shape (13233, 2914)
        Each row corresponds to a ravelled face image of original size 62 x 47
        pixels. Changing the ``slice_`` or resize parameters will change the shape
        of the output.

    dataset.images : numpy array of shape (13233, 62, 47)
        Each row is a face image corresponding to one of the 5749 people in
        the dataset. Changing the ``slice_`` or resize parameters will change the shape
        of the output.

    dataset.target : numpy array of shape (13233,)
        Labels associated to each face image. Those labels range from 0-5748
        and correspond to the person IDs.

    dataset.DESCR : string
        Description of the Labeled Faces in the Wild (LFW) dataset.
    """
    lfw_home, data_folder_path = check_fetch_lfw(
        data_home=data_home, funneled=funneled,
        download_if_missing=download_if_missing)
    logger.info('Loading LFW people faces from %s', lfw_home)

    # wrap the loader in a memoizing function that will return memmaped data
    # arrays for optimal memory usage
    m = Memory(cachedir=lfw_home, compress=6, verbose=0)
    load_func = m.cache(_fetch_lfw_people)

    # load and memoize the pairs as np arrays
    faces, target, target_names = load_func(
        data_folder_path, resize=resize,
        min_faces_per_person=min_faces_per_person, color=color, slice_=slice_)

    # pack the results as a Bunch instance
    return Bunch(data=faces.reshape(len(faces), -1), images=faces,
                 target=target, target_names=target_names,
                 DESCR="LFW faces dataset")

#
# Task #2:  Face Verification on pairs of face pictures
#

def _fetch_lfw_pairs(index_file_path, data_folder_path, slice_=None,
                     color=False, resize=None):
    """Perform the actual data loading for the LFW pairs dataset

    This operation is meant to be cached by a joblib wrapper.
    """
    # parse the index file to find the number of pairs to be able to allocate
    # the right amount of memory before starting to decode the jpeg files
    with open(index_file_path, 'rb') as index_file:
        split_lines = [ln.strip().split(b('\t')) for ln in index_file]
    pair_specs = [sl for sl in split_lines if len(sl) > 2]
    n_pairs = len(pair_specs)

    # interating over the metadata lines for each pair to find the filename to
    # decode and load in memory
    target = np.zeros(n_pairs, dtype=np.int)
    file_paths = list()
    for i, components in enumerate(pair_specs):
        if len(components) == 3:
            target[i] = 1
            pair = (
                (components[0], int(components[1]) - 1),
                (components[0], int(components[2]) - 1),
            )
        elif len(components) == 4:
            target[i] = 0
            pair = (
                (components[0], int(components[1]) - 1),
                (components[2], int(components[3]) - 1),
            )
        else:
            raise ValueError("invalid line %d: %r" % (i + 1, components))
        for j, (name, idx) in enumerate(pair):
            try:
                person_folder = join(data_folder_path, name)
            except TypeError:
                person_folder = join(data_folder_path, str(name, 'UTF-8'))
            filenames = list(sorted(listdir(person_folder)))
            file_path = join(person_folder, filenames[idx])
            file_paths.append(file_path)

    pairs = _load_imgs(file_paths, slice_, color, resize)
    shape = list(pairs.shape)
    n_faces = shape.pop(0)
    shape.insert(0, 2)
    shape.insert(0, n_faces // 2)
    pairs.shape = shape

    return pairs, target, np.array(['Different persons', 'Same person'])

@deprecated("Function 'load_lfw_people' has been deprecated in 0.17 and will be "
            "removed in 0.19."
            "Use fetch_lfw_people(download_if_missing=False) instead.")
def load_lfw_people(download_if_missing=False, **kwargs):
    """Alias for fetch_lfw_people(download_if_missing=False)

    Check fetch_lfw_people.__doc__ for the documentation and parameter list.
    """
    return fetch_lfw_people(download_if_missing=download_if_missing, **kwargs)

def fetch_lfw_pairs(subset='train', data_home=None, funneled=True, resize=0.5,
                    color=False, slice_=(slice(70, 195), slice(78, 172)),
                    download_if_missing=True):
    """Loader for the Labeled Faces in the Wild (LFW) pairs dataset

    This dataset is a collection of JPEG pictures of famous people
    collected on the internet, all details are available on the
    official website:

        http://vis-www.cs.umass.edu/lfw/

    Each picture is centered on a single face. Each pixel of each channel
    (color in RGB) is encoded by a float in range 0.0 - 1.0.

    The task is called Face Verification: given a pair of two pictures,
    a binary classifier must predict whether the two images are from
    the same person.

    In the official `README.txt`_ this task is described as the
    "Restricted" task.  As I am not sure as to implement the
    "Unrestricted" variant correctly, I left it as unsupported for now.

      .. _`README.txt`: http://vis-www.cs.umass.edu/lfw/README.txt

    The original images are 250 x 250 pixels, but the default slice and resize
    arguments reduce them to 62 x 74.

    Read more in the :ref:`User Guide <labeled_faces_in_the_wild>`.

    Parameters
    ----------
    subset : optional, default: 'train'
        Select the dataset to load: 'train' for the development training
        set, 'test' for the development test set, and '10_folds' for the
        official evaluation set that is meant to be used with a 10-folds
        cross validation.

    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By
        default all scikit learn data is stored in '~/scikit_learn_data'
        subfolders.

    funneled : boolean, optional, default: True
        Download and use the funneled variant of the dataset.

    resize : float, optional, default 0.5
        Ratio used to resize the each face picture.

    color : boolean, optional, default False
        Keep the 3 RGB channels instead of averaging them to a single
        gray level channel. If color is True the shape of the data has
        one more dimension than than the shape with color = False.

    slice_ : optional
        Provide a custom 2D slice (height, width) to extract the
        'interesting' part of the jpeg files and avoid use statistical
        correlation from the background

    download_if_missing : optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    The data is returned as a Bunch object with the following attributes:

    data : numpy array of shape (2200, 5828)
        Each row corresponds to 2 ravel'd face images of original size 62 x 47
        pixels. Changing the ``slice_`` or resize parameters will change the shape
        of the output.

    pairs : numpy array of shape (2200, 2, 62, 47)
        Each row has 2 face images corresponding to same or different person
        from the dataset containing 5749 people. Changing the ``slice_`` or resize
        parameters will change the shape of the output.

    target : numpy array of shape (13233,)
        Labels associated to each pair of images. The two label values being
        different persons or the same person.

    DESCR : string
        Description of the Labeled Faces in the Wild (LFW) dataset.

    """
    lfw_home, data_folder_path = check_fetch_lfw(
        data_home=data_home, funneled=funneled,
        download_if_missing=download_if_missing)
    logger.info('Loading %s LFW pairs from %s', subset, lfw_home)

    # wrap the loader in a memoizing function that will return memmaped data
    # arrays for optimal memory usage
    m = Memory(cachedir=lfw_home, compress=6, verbose=0)
    load_func = m.cache(_fetch_lfw_pairs)

    # select the right metadata file according to the requested subset
    label_filenames = {
        'train': 'pairsDevTrain.txt',
        'test': 'pairsDevTest.txt',
        '10_folds': 'pairs.txt',
    }
    if subset not in label_filenames:
        raise ValueError("subset='%s' is invalid: should be one of %r" % (
            subset, list(sorted(label_filenames.keys()))))
    index_file_path = join(lfw_home, label_filenames[subset])

    # load and memoize the pairs as np arrays
    pairs, target, target_names = load_func(
        index_file_path, data_folder_path, resize=resize, color=color,
        slice_=slice_)

    # pack the results as a Bunch instance
    return Bunch(data=pairs.reshape(len(pairs), -1), pairs=pairs,
                 target=target, target_names=target_names,
                 DESCR="'%s' segment of the LFW pairs dataset" % subset)

@deprecated("Function 'load_lfw_pairs' has been deprecated in 0.17 and will be "
            "removed in 0.19."
            "Use fetch_lfw_pairs(download_if_missing=False) instead.")
def load_lfw_pairs(download_if_missing=False, **kwargs):
    """Alias for fetch_lfw_pairs(download_if_missing=False)

    Check fetch_lfw_pairs.__doc__ for the documentation and parameter list.
    """
    return fetch_lfw_pairs(download_if_missing=download_if_missing, **kwargs)