# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Methods to allow pandas.DataFrame."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

try:
  # pylint: disable=g-import-not-at-top
  import pandas as pd
  HAS_PANDAS = True
except ImportError:
  HAS_PANDAS = False

PANDAS_DTYPES = {
    'int8': 'int',
    'int16': 'int',
    'int32': 'int',
    'int64': 'int',
    'uint8': 'int',
    'uint16': 'int',
    'uint32': 'int',
    'uint64': 'int',
    'float16': 'float',
    'float32': 'float',
    'float64': 'float',
    'bool': 'i'
}

def extract_pandas_data(data):
  """Extract data from pandas.DataFrame for predictors.

  Given a DataFrame, will extract the values and cast them to float. The
  DataFrame is expected to contain values of type int, float or bool.

  Args:
    data: `pandas.DataFrame` containing the data to be extracted.

  Returns:
    A numpy `ndarray` of the DataFrame's values as floats.

  Raises:
    ValueError: if data contains types other than int, float or bool.
  """
  if not isinstance(data, pd.DataFrame):
    return data

  bad_data = [column for column in data
              if data[column].dtype.name not in PANDAS_DTYPES]

  if not bad_data:
    return data.values.astype('float')
  else:
    error_report = [("'" + str(column) + "' type='" +
                     data[column].dtype.name + "'") for column in bad_data]
    raise ValueError('Data types for extracting pandas data must be int, '
                     'float, or bool. Found: ' + ', '.join(error_report))

def extract_pandas_matrix(data):
  """Extracts numpy matrix from pandas DataFrame.

  Args:
    data: `pandas.DataFrame` containing the data to be extracted.

  Returns:
    A numpy `ndarray` of the DataFrame's values.
  """
  if not isinstance(data, pd.DataFrame):
    return data

  return data.as_matrix()

def extract_pandas_labels(labels):
  """Extract data from pandas.DataFrame for labels.

  Args:
    labels: `pandas.DataFrame` or `pandas.Series` containing one column of
      labels to be extracted.

  Returns:
    A numpy `ndarray` of labels from the DataFrame.

  Raises:
    ValueError: if more than one column is found or type is not int, float or
      bool.
  """
  if isinstance(labels,
                pd.DataFrame):  # pandas.Series also belongs to DataFrame
    if len(labels.columns) > 1:
      raise ValueError('Only one column for labels is allowed.')

    bad_data = [column for column in labels
                if labels[column].dtype.name not in PANDAS_DTYPES]
    if not bad_data:
      return labels.values
    else:
      error_report = ["'" + str(column) + "' type="
                      + str(labels[column].dtype.name) for column in bad_data]
      raise ValueError('Data types for extracting labels must be int, '
                       'float, or bool. Found: ' + ', '.join(error_report))
  else:
    return labels


# defining the base filter curve classes

import os

from scipy import interpolate
from wsynphot.spectrum1d import SKSpectrum1D as Spectrum1D
import pandas as pd
from wsynphot.io.cache_filters import load_filter_index, load_transmission_data

from astropy import units as u, constants as const

from astropy import utils
import numpy as np
from wsynphot.calibration import get_vega_calibration_spectrum

def calculate_filter_flux_density(spectrum, filter):
    """
    Calculate the average flux through the filter by evaluating the integral

    ..math::

        f_lambda = \\frac{\\int_}{}
    Parameters
    ----------

    spectrum: ~specutils.Spectrum1D
        spectrum object
    filter: ~wsynphot.FilterCurve

    :return:
    """

    filtered_spectrum = filter * spectrum
    filter_flux_density = np.trapz(filtered_spectrum.flux * filtered_spectrum.wavelength,
                    filtered_spectrum.wavelength)
    return filter_flux_density

def calculate_vega_magnitude(spectrum, filter):
    filter_flux_density = calculate_filter_flux_density(spectrum, filter)
    wavelength_delta = filter.calculate_wavelength_delta()
    filtered_f_lambda = (filter_flux_density / wavelength_delta)

    zp_vega_f_lambda = filter.zp_vega_f_lambda

    return -2.5 * np.log10(filtered_f_lambda / zp_vega_f_lambda)

def calculate_ab_magnitude(spectrum, filter):
    filtered_f_lambda = (calculate_filter_flux_density(spectrum, filter) /
                         filter.calculate_wavelength_delta())

    return -2.5 * np.log10(filtered_f_lambda / filter.zp_ab_f_lambda)

def list_filters():
    """
    List available filter sets along with their properties
    """
    return load_filter_index()

class BaseFilterCurve(object):
    """
    Basic filter curve class

    Parameters
    ----------

    wavelength: ~astropy.units.Quantity
        wavelength for filter curve

    transmission_lambda: numpy.ndarray
        transmission_lambda for filter curve

    interpolation_kind: str
        allowed interpolation kinds given in scipy.interpolate.interp1d
    """

    @classmethod
    def load_filter(cls, filter_id=None, interpolation_kind='linear'):
        """

        Parameters
        ----------

        filter_id: str or None
            if None is provided will return a DataFrame of all filters

        interpolation_kind: str
            see scipy.interpolation.interp1d

        """
        if filter_id is None:
            return list_filters()

        else:
            filter = load_transmission_data(filter_id)

            wavelength_unit = 'angstrom'

            wavelength = filter['Wavelength'].values * u.Unit(wavelength_unit)

            return cls(wavelength, filter['Transmission'].values,
                       interpolation_kind=interpolation_kind,
                       filter_id=filter_id)

    def __init__(self, wavelength, transmission_lambda,
                 interpolation_kind='linear', filter_id=None):
        if not hasattr(wavelength, 'unit'):
            raise ValueError('the wavelength needs to be a astropy quantity')
        self.wavelength = wavelength
        self.transmission_lambda = transmission_lambda

        self.interpolation_object = interpolate.interp1d(self.wavelength,
                                                         self.transmission_lambda,
                                                         kind=interpolation_kind,
                                                         bounds_error=False,
                                                         fill_value=0.0)
        self.filter_id = filter_id

    def __mul__(self, other):
        if not hasattr(other, 'flux') or not hasattr(other, 'wavelength'):
            raise ValueError('requiring a specutils.Spectrum1D-like object that'
                             'has attributes "flux" and "wavelength"')

        #new_wavelength = np.union1d(other.wavelength.to(self.wavelength.unit).value,
        #                            self.wavelength.value) * self.wavelength.unit
        transmission = self.interpolate(other.wavelength)

        return Spectrum1D.from_array(other.wavelength, transmission * other.flux)

    def __rmul__(self, other):
        return self.__mul__(other)

    @utils.lazyproperty
    def lambda_pivot(self):
        """
        Calculate the pivotal wavelength as defined in Bessell & Murphy 2012

        .. math::

            \\lambda_\\textrm{pivot} = \\sqrt{
            \\frac{\\int S(\\lambda)\\lambda d\\lambda}{\\int \\frac{S(\\lambda)}{\\lambda}}}\\\\
            <f_\\nu> = <f_\\lambda>\\frac{\\lambda_\\textrm{pivot}^2}{c}
        """

        return np.sqrt((np.trapz(self.transmission_lambda * self.wavelength, self.wavelength)/
                (np.trapz(self.transmission_lambda / self.wavelength, self.wavelength))))

    @utils.lazyproperty
    def wavelength_start(self):
        return self.get_wavelength_start()

    @utils.lazyproperty
    def wavelength_end(self):
        return self.get_wavelength_end()

    @utils.lazyproperty
    def zp_ab_f_lambda(self):
        return (self.zp_ab_f_nu * const.c / self.lambda_pivot**2).to(
            'erg/s/cm^2/Angstrom', u.spectral())

    @utils.lazyproperty
    def zp_ab_f_nu(self):
        return (3631 * u.Jy).to('erg/s/cm^2/Hz')

    @utils.lazyproperty
    def zp_vega_f_lambda(self):
        return (calculate_filter_flux_density(get_vega_calibration_spectrum(), self) /
                self.calculate_wavelength_delta())

    def interpolate(self, wavelength):
        """
        Interpolate the filter onto new wavelength grid

        Parameters
        ----------

        wavelength: ~astropy.units.Quantity
            wavelength grid to interpolate on

        """

        converted_wavelength = wavelength.to(self.wavelength.unit)
        return self.interpolation_object(converted_wavelength)

    def _calculuate_flux_density(self, wavelength, flux):
        return _calculcate_filter_flux_density(flux, self)

    def calculate_flux_density(self, spectrum):
        return calculate_filter_flux_density(spectrum, self)

    def calculate_f_lambda(self, spectrum):
        return (self.calculate_flux_density(spectrum) /
                self.calculate_wavelength_delta())

    def calculate_wavelength_delta(self):
        """
        Calculate the Integral :math:`\integral
        :return:
        """

        return np.trapz(self.transmission_lambda * self.wavelength,
                        self.wavelength)

    def calculate_weighted_average_wavelength(self):
        """
        Calculate integral :math:`\\frac{\\int S(\\lambda) \\lambda d\\lambda}{\\int S(\\lambda) d\\lambda}`

        Returns
            : ~astropy.units.Quantity

        """

        return (np.trapz(self.transmission_lambda * self.wavelength,
                         self.wavelength) / self.calculate_wavelength_delta())

    def calculate_vega_magnitude(self, spectrum):
        __doc__ = calculate_vega_magnitude.__doc__
        return calculate_vega_magnitude(spectrum, self)

    def calculate_ab_magnitude(self, spectrum):
        __doc__ = calculate_ab_magnitude.__doc__
        return calculate_ab_magnitude(spectrum, self)

    def convert_ab_magnitude_to_f_lambda(self, mag):
        return 10**(-0.4*mag) * self.zp_ab_f_lambda

    def convert_vega_magnitude_to_f_lambda(self, mag):
        return 10**(-0.4*mag) * self.zp_vega_f_lambda

    def plot(self, ax, scale_max=None, make_label=True, plot_kwargs={},
             format_filter_id=None):
        if scale_max is not None:
            if hasattr(scale_max, 'unit'):
                scale_max = scale_max.value

            transmission = (self.transmission_lambda * scale_max
                            / self.transmission_lambda.max())
        else:
            transmission = self.transmission_lambda

        ax.plot(self.wavelength, transmission, **plot_kwargs)
        ax.set_xlabel('Wavelength [{0}]'.format(
            self.wavelength.unit.to_string(format='latex')))
        ax.set_ylabel('Transmission [1]')

        if make_label==True and self.filter_id is not None:
            if format_filter_id is not None:
                filter_id = format_filter_id(self.filter_id)
            else:
                filter_id = self.filter_id
            text_x = (self.lambda_pivot).value
            text_y = transmission.max()/2
            ax.text(text_x, text_y, filter_id,
                    horizontalalignment='center', verticalalignment='center',
                    bbox=dict(facecolor='white', alpha=0.5))

    def get_wavelength_start(self, threshold=0.01):
        norm_cum_sum = (np.cumsum(self.transmission_lambda)
                        / np.sum(self.transmission_lambda))
        return self.wavelength[norm_cum_sum.searchsorted(threshold)]

    def get_wavelength_end(self, threshold=0.01):
        norm_cum_sum = (np.cumsum(self.transmission_lambda)
                        / np.sum(self.transmission_lambda))
        return self.wavelength[norm_cum_sum.searchsorted(1 - threshold)]

class FilterCurve(BaseFilterCurve):
    def __repr__(self):
        if self.filter_id is None:
            filter_id = "{0:x}".format(self.__hash__())
        else:
            filter_id = self.filter_id
        return "FilterCurve <{0}>".format(filter_id)

class FilterSet(object):

    """
    A set of filters

    Parameters
    ----------

    filter_set: ~list
        a list of strings or a list of filters

    interpolation_kind: ~str
        scipy interpolaton kinds

    """
    def __init__(self, filter_set, interpolation_kind='linear'):

        if hasattr(filter_set[0], 'wavelength'):
            self.filter_set = filter_set
        else:
            self.filter_set = [FilterCurve.load_filter(filter_id,
                                              interpolation_kind=
                                              interpolation_kind)
                      for filter_id in filter_set]

    def __iter__(self):
        self.current_filter_idx = 0
        return self

    def __next__(self):
        try:
            item = self.filter_set[self.current_filter_idx]
        except IndexError:
            raise StopIteration

        self.current_filter_idx += 1
        return item
    next = __next__

    def __getitem__(self, item):
        return self.filter_set.__getitem__(item)

    def __repr__(self):
        return "<{0} \n{1}>".format(self.__class__.__name__,
                                    '\n'.join(
                                        [item.filter_id
                                         for item in self.filter_set]))

    @property
    def lambda_pivot(self):
        return u.Quantity([item.lambda_pivot for item in self])

    def calculate_f_lambda(self, spectrum):
        return u.Quantity(
            [item.calculate_f_lambda(spectrum) for item in self.filter_set])

    def calculate_ab_magnitudes(self, spectrum):
        mags = [item.calculate_ab_magnitude(spectrum)
                for item in self.filter_set]
        return mags

    def calculate_vega_magnitudes(self, spectrum):
        mags = [item.calculate_vega_magnitude(spectrum)
                for item in self.filter_set]
        return mags

    def convert_ab_magnitudes_to_f_lambda(self, magnitudes):
        if len(magnitudes) != len(self.filter_set):
            raise ValueError("Filter set and magnitudes need to have the same "
                             "number of items")
        f_lambdas = [filter.convert_ab_magnitude_to_f_lambda(mag)
                     for filter, mag in zip(self.filter_set, magnitudes)]
        return u.Quantity(f_lambdas)

    def convert_ab_magnitude_uncertainties_to_f_lambda_uncertainties(
            self, magnitudes, magnitude_uncertainties):

        if len(magnitudes) != len(self.filter_set):
            raise ValueError("Filter set and magnitudes need to have the same "
                             "number of items")
        f_lambda_positive_uncertainties = u.Quantity(
            [filter.convert_ab_magnitude_to_f_lambda(mag +  mag_uncertainty)
                     for filter, mag, mag_uncertainty in zip(
                self.filter_set, magnitudes, magnitude_uncertainties, )])

        f_lambda_negative_uncertainties = u.Quantity(
            [filter.convert_ab_magnitude_to_f_lambda(mag -  mag_uncertainty)
                     for filter, mag, mag_uncertainty in zip(
                self.filter_set, magnitudes, magnitude_uncertainties)])

        return np.abs(u.Quantity((f_lambda_positive_uncertainties,
                f_lambda_negative_uncertainties))
                      -  self.convert_ab_magnitudes_to_f_lambda(magnitudes))

    def convert_vega_magnitude_uncertainties_to_f_lambda_uncertainties(
            self, magnitudes, magnitude_uncertainties):

        if len(magnitudes) != len(self.filter_set):
            raise ValueError("Filter set and magnitudes need to have the same "
                             "number of items")
        f_lambda_positive_uncertainties = u.Quantity(
            [filter.convert_vega_magnitude_to_f_lambda(mag +  mag_uncertainty)
                     for filter, mag, mag_uncertainty in zip(
                self.filter_set, magnitudes, magnitude_uncertainties, )])

        f_lambda_negative_uncertainties = u.Quantity(
            [filter.convert_vega_magnitude_to_f_lambda(mag -  mag_uncertainty)
                     for filter, mag, mag_uncertainty in zip(
                self.filter_set, magnitudes, magnitude_uncertainties)])

        return np.abs(u.Quantity((f_lambda_positive_uncertainties,
                                  f_lambda_negative_uncertainties))
                      -  self.convert_vega_magnitudes_to_f_lambda(magnitudes))

    def convert_vega_magnitudes_to_f_lambda(self, magnitudes):
        if len(magnitudes) != len(self.filter_set):
            raise ValueError("Filter set and magnitudes need to have the same "
                             "number of items")
        f_lambdas = [filter.convert_vega_magnitude_to_f_lambda(mag)
                     for filter, mag in zip(self.filter_set, magnitudes)]
        return u.Quantity(f_lambdas)

    def plot_spectrum(self, spectrum, ax, make_labels=True,
                      spectrum_plot_kwargs={}, filter_plot_kwargs={},
                      filter_color_list=None, format_filter_id=None):
        """
        plot a spectrum with the given filters
        spectrum:
        ax:
        make_labels:
        :return:
        """
        ax.plot(spectrum.wavelength, spectrum.flux, **spectrum_plot_kwargs)
        for i, filter in enumerate(self.filter_set):
            filter_scale = filter.calculate_f_lambda(spectrum)
            if filter_color_list is not None:
                filter_plot_kwargs['color'] = filter_color_list[i]
            filter.plot(ax, scale_max=filter_scale, make_label=make_labels,
                        plot_kwargs=filter_plot_kwargs,
                        format_filter_id=format_filter_id)

class MagnitudeSet(FilterSet):
    def __init__(self, filter_set, magnitudes, magnitude_uncertainties=None,
                 interpolation_kind='linear'):
        super(MagnitudeSet, self).__init__(filter_set,
                                           interpolation_kind=
                                           interpolation_kind)
        self.magnitudes = np.array(magnitudes)
        self.magnitude_uncertainties = np.array(magnitude_uncertainties)

    def __repr__(self):
        mag_str = '{0} {1:.4f} +/- {2:.4f}'
        mag_data = []
        for i, filter in enumerate(self.filter_set):
            unc = (np.nan if self.magnitude_uncertainties is None
                   else self.magnitude_uncertainties[i])
            mag_data.append(mag_str.format(filter.filter_id,
                                           self.magnitudes[i], unc))

        return "<{0} \n{1}>".format(self.__class__.__name__,
                                    '\n'.join(mag_data))


# -*- coding: utf-8 -*-
#!/usr/bin/python
#
# Author    Yann Bayle
# E-mail    bayle.yann@live.fr
# License   MIT
# Created   13/10/2016
# Updated   20/01/2017
# Version   1.0.0
#

"""
Description of svmbff.py
======================

bextract -mfcc -zcrs -ctd -rlf -flx -ws 1024 -as 898 -sv -fe filename.mf -w out.arff

:Example:

python svmbff.py
"""

import os
import csv
import sys
import time
import utils
import shutil
import argparse
import multiprocessing
from statistics import stdev
from scipy.io import arff
from sklearn.metrics import precision_recall_curve, precision_score, recall_score, classification_report, f1_score, accuracy_score

begin = int(round(time.time() * 1000))

def validate_arff(filename):
    """Description of validate_arff

    Check if filename exists on path and is a file
    If file corresponds to valid arff file return absolute path
    Otherwise move file to invalid directory and return False
    """
    # Check if file exists
    if os.path.isfile(filename) and os.path.exists(filename):
        filename = os.path.abspath(filename)
    else:
        return False
    # If does not satisfy min size, move to "empty" folder
    if os.stat(filename).st_size < 8100:
        tmp_path = filename.split("/")
        empty_dirname = "/".join(tmp_path[:-1]) + "/empty/"
        if not os.path.exists(empty_dirname):
            os.makedirs(empty_dirname)
        shutil.move(filename, empty_dirname + tmp_path[-1])
        return False
    # # If filename does not match with feature name, move to "invalid" folder
    # name_file = filename.split("/")[-1][:12]
    # with open(filename) as filep:
    #     for i, line in enumerate(filep):
    #         if i == 70:
    #             # 71th line
    #             name_feat = line.split(" ")[2][1:13]
    #             break
    # if name_file != name_feat:
    #     tmp_path = filename.split("/")
    #     invalid_dirname = "/".join(tmp_path[:-1]) + "/invalid/"
    #     if not os.path.exists(invalid_dirname):
    #         os.makedirs(invalid_dirname)
    #     shutil.move(filename, invalid_dirname + tmp_path[-1])
    #     return False
    # If everything went well, return filename absolute path
    return filename

def merge_arff(indir, outfilename):
    """Description of merge_arff

    bextract program from Marsyas generate one output file per audio file
    This function merge them all in one unique file
    Check if analysed file are valid i.e. not empty
    """
    utils.print_success("Preprocessing ARFFs")
    indir = utils.abs_path_dir(indir)
    filenames = os.listdir(indir)
    outfn = open(outfilename, 'w')
    cpt_invalid_fn = 0
    # Write first lines of ARFF template file
    for filename in filenames:
        if os.path.isfile(indir + filename):
            new_fn = validate_arff(indir + filename)
            if new_fn:
                with open(new_fn, 'r') as template:
                    nb_line = 74
                    for line in template:
                        if not nb_line:
                            break
                        nb_line -= 1
                        outfn.write(line)
                    break
            else:
                cpt_invalid_fn += 1
    # Append all arff file to the output file
    cur_file_num = 1
    for filename in filenames:
        if os.path.isfile(indir + filename):
            new_fn = validate_arff(indir + filename)
            if new_fn:
                cur_file_num = cur_file_num + 1
                utils.print_progress_start("Analysing file\t" + str(cur_file_num))
                fname = open(new_fn, 'r')
                outfn.write("".join(fname.readlines()[74:77]))
                fname.close()
            else:
                cpt_invalid_fn += 1
    utils.print_progress_end()
    outfn.close()
    # os.system("rm " + indir + "*.arff")
    if cpt_invalid_fn:
        utils.print_warning(str(cpt_invalid_fn) + " ARFF files with errors found")
    return outfilename

def add_groundtruth(feature_fn, groundtruth_fn, output_fn):
    """Description of add_groundtruth

    Write in output filename the groundtruth merged with corresponding features

    ..todo:: Error with old_tag not corresponding to filename...

    """
    utils.print_success("Adding groundtruth")
    feature_fn = utils.abs_path_file(feature_fn)
    groundtruth_fn = utils.abs_path_file(groundtruth_fn)
    if os.path.isfile(output_fn) and os.path.exists(output_fn):
        utils.print_warning("Overwritting existing output file: " +
            utils.abs_path_file(output_fn))
    # TODO Read groundtruth file in memory
    tmp_gt = csv.reader(open(groundtruth_fn, "r"))
    groundtruths = {}
    for row in tmp_gt:
        groundtruths[row[0]] = row[1]
    tags = []
    output = open(output_fn, "w")

    # switch if test set preprocessing
    # separator = "_"
    separator = "."

    with open(feature_fn, "r") as feat:
        line_num = 0
        tmp_line = ""
        for line in feat:
            line_num += 1
            if line_num > 74:
                if line[0] != "%":
                    # Alter feature line with correct tag
                    cur_line = line.split(",")
                    old_tag = cur_line[-1].split(separator)[0]
                    if old_tag in groundtruths:
                        new_tag = groundtruths[old_tag]
                        output.write(tmp_line + ",".join(cur_line[:-1]) + "," + new_tag +"\n")
                        tmp_line = ""
                        tags.append(new_tag)
                    else:
                        # TODO
                        # File not in groundtruth
                        tmp_line = ""
                        # utils.print_warning("Error with " + old_tag)
                else:
                    tmp_line += line
            elif line_num == 2:
                output.write("@relation train_test.arff\n")
                # output.write("@relation MARSYAS_KEA\n")
            elif line_num == 71:
                # Alter line 71 containing all tag gathered along the way
                # TODO enhance
                output.write("@attribute output {i,s}\n")
            else:
                # Write header
                output.write(line)
    output.close()

def split_number(number, nb_folds):
    """Description of split_number

    Return an int array of size nb_folds where the sum of cells = number
    All the integers in cells are the same +-1
    """
    if not isinstance(number, int) and not isinstance(nb_folds, int):
        utils.print_error("Variable must be integer")
    if number < nb_folds:
        utils.print_error("Number of folds > Number of data available")
    min_num = int(number/nb_folds)
    folds = [min_num] * nb_folds
    for num in range(0, number-(min_num*nb_folds)):
        folds[num] = folds[num] + 1
    return folds

def create_folds(filelist, nb_folds, folds_dir, invert_train_test=False):
    """Description of create_folds

    """
    utils.print_success("Creating folds")
    if nb_folds < 1:
        utils.print_error("Wrong number of folds provided")

    # folds_dir = "/".join(filelist.split("/")[:-1])
    if nb_folds == 1:
        # Train and test set are the same
        folds_dir = folds_dir + "01_fold/"
        utils.create_dir(folds_dir)
        os.system("cp " + filelist + " " + folds_dir + "/train_test.arff")
    else:
        # Create train and test set
        folds_dir = folds_dir + str(nb_folds).zfill(2) + "_folds/"
        utils.create_dir(folds_dir)
        # TODO
        # Read filelist
        # Extract name and tag
        # Separate different tag
        # create folds
        data, meta = arff.loadarff(filelist)
        tags = {}
        for row in data:
            tag = row[-1].decode("ascii")
            if tag in tags:
                tags[tag] += 1
            else:
                tags[tag] = 1
        tags_folds = {}
        tags_folds_index = {}
        for tag in tags:
            tags_folds[tag] = split_number(tags[tag], nb_folds)
            tags_folds_index[tag] = 0
        # Create empty folds
        folds = {}
        # Init empty folds
        for index in range(0, nb_folds):
            folds[index] = ""
        # Fill folds with data
        with open(filelist, "r") as filelist_pointer:
            arff_header = ""
            tmp = ""
            for i, line in enumerate(filelist_pointer):
                utils.print_progress_start("\t" + str(i))
                # Until the 75th line
                if i > 74:
                    # Process ARFF data
                    if "% " in line:
                        # Memorize line
                        tmp += line
                    else:
                        # Get line 3 and add it to corresponding fold
                        tag = line.split(",")[-1][:-1]
                        num_fold = tags_folds_index[tag]
                        if tags_folds[tag][num_fold] == 0:
                            tags_folds_index[tag] += 1
                        tags_folds[tag][tags_folds_index[tag]] -= 1
                        folds[tags_folds_index[tag]] += tmp + line
                        tmp = ""
                else:
                    # Save ARFF header lines
                    arff_header += line
            utils.print_progress_end
        # At this point data has been split up in different part
        # Use this part to create train/test split
        if invert_train_test:
            # Test is bigger than train
            fn_with_min_data = "/train_"
            fn_with_max_data = "/test_"
        else:
            # Train is bigger than test
            fn_with_min_data = "/test_"
            fn_with_max_data = "/train_"
        for index_test in range(0, nb_folds):
            filep = open(folds_dir + fn_with_min_data + str(index_test+1).zfill(2) + ".arff", "a")
            filep.write(arff_header + folds[index_test])
            filep.close()
            filep = open(folds_dir + fn_with_max_data + str(index_test+1).zfill(2) + ".arff", "a")
            filep.write(arff_header)
            for index_train in range(0, nb_folds):
                if index_train != index_test:
                    filep.write(folds[index_train])
            filep.close()
    return folds_dir

def process_results(in_fn, out_fn):
    in_fn = utils.abs_path_file(in_fn)
    out_fp = open(out_fn, "w")
    with open(in_fn, "r") as filep:
        for index, line in enumerate(filep):
            if index % 2:
                row = line[:-1].split("\t")
                out_fp.write(row[0].split("_")[0] + "," + row[2] + "\n")
    out_fp.close()

def experiment_2_3():
    process_results("src/tmp/svmbff/SVMBFF.csv", "predictions/SVMBFF.csv")

def run_kea(train_file, test_file, out_file, verbose=False):
    """Description of run_kea

    Launch kea classification on specified file
    """
    kea_cmd = 'kea -m tags -w ' + train_file + ' -tw ' + test_file + ' -pr ' + out_file
    if not verbose:
        kea_cmd += "> /dev/null 2>&1"
    os.system(kea_cmd)
    train_dir = train_file.split(os.sep)
    train_dir = os.sep.join(train_dir[:-1])
    # os.system("rm " + train_dir + "/*affinities*")
    test_dir = test_file.split(os.sep)
    test_dir = os.sep.join(test_dir[:-1])
    # os.system("rm " + test_dir + "/*affinities*")

def run_kea_on_folds(folds_dir):
    """Description of run_kea_on_folds

    Wrapper for kea on folds
    """
    folds_dir = utils.abs_path_dir(folds_dir)
    out_file = folds_dir + "/results.txt"
    if os.path.exists(folds_dir + "/train_test.arff"):
        train_file = folds_dir + "/train_test.arff"
        test_file = train_file
        run_kea(train_file, test_file, out_file)
    else:
        nb_folds = len([name for name in os.listdir(folds_dir) if os.path.isfile(os.path.join(folds_dir, name))])
        # Run on multiple train/test
        for index in range(1, int(nb_folds/2)+1):
            utils.print_progress_start("Train/Test on fold " + str(index))
            train_file = folds_dir + "/train_" + str(index).zfill(2) + ".arff"
            test_file = folds_dir + "/test_" + str(index).zfill(2) + ".arff"
            out_file = folds_dir + "/results_" + str(index).zfill(2) + ".arff"
            run_kea(train_file, test_file, out_file)
        utils.print_progress_end()
        utils.print_warning("TODO multiprocessing")
        # # Parallel computing on each TrainTestFolds
        # printTitle("Parallel train & test of folds")
        # partialRunTrainTestOnFold = partial(runTrainTestOnFold, args=args)
        # pool = multiprocessing.Pool()
        # pool.map(partialRunTrainTestOnFold, range(nb_folds)) #make our results with a map call
        # pool.close() #we are not adding any more processes
        # pool.join() #tell it to wait until all threads are done before going on

def extract_feat_train():
    dirs = ["/media/sf_SharedFolder/DataSets/Jamendo/Yann/song/",
            "/media/sf_SharedFolder/DataSets/ccmixter_corpus/instru/",
            "/media/sf_SharedFolder/DataSets/MedleyDB/MedleyDB/instru/vrai/"]
    outdir= "res/"
    for indir in dirs:
        extensions = ["wav", "mp3"]
        filenames = [fn for fn in os.listdir(indir)
                if any(fn.endswith(ext) for ext in extensions)]
        for index, filename in enumerate(filenames):
            dirName = indir.split("/")[-2] + ".mf"
            with open(dirName, "w") as filep:
                filep.write(indir + filename + "\n")
            outfilename = outdir + filename[:-3].replace(" ", "_") + "arff"
            bextract_cmd = "bextract -mfcc -zcrs -ctd -rlf -flx -ws 1024 -as 898 -sv -fe " + dirName + " -w " + outfilename
            os.system(bextract_cmd)

def read_gts(filename):
    filename = utils.abs_path_file(filename)
    groundtruths = {}
    i = 0
    with open(filename, "r") as filep:
        for index, line in enumerate(filep):
            if index > 73:
                if i == 0:
                    i += 1
                    name = line.split("/")[-1][:-1]
                elif i == 1:
                    i += 1
                elif i == 2:
                    i = 0
                    groundtruths[name] = line.split(",")[-1][:-1]
    return groundtruths

def read_preds(filename):
    pres_filen = utils.abs_path_file(filename)
    predictions = {}
    i = 0
    with open(filename, "r") as filep:
        for index, line in enumerate(filep):
            if index % 2:
                line = line.split("\t")
                name = line[0].split("/")[-1]
                pred = float(line[-1])
                if pred > 0.5:
                    predictions[name] = "s"
                else:
                    predictions[name] = "i"
    return predictions

def figure2():
    # folds_dir = create_folds("results/dataset.arff", 5)
    # run_kea_on_folds(folds_dir)
    # read results arff file and print accuracy and f-measure
    gts_filen = "results/dataset.arff"
    gts = read_gts(gts_filen)
    folds_dir = "results/05_folds/"
    res_files = [name for name in os.listdir(folds_dir) if os.path.isfile(os.path.join(folds_dir, name)) and "results" in name]
    acc = []
    f1 = []
    for res in res_files:
        predictions = []
        groundtruths = []
        preds = read_preds(folds_dir + res)
        for name in preds:
            if name in gts:
                groundtruths.append(gts[name])
                predictions.append(preds[name])
        acc.append(accuracy_score(groundtruths, predictions))
        predictions = [1 if i=="s" else 0 for i in predictions]
        groundtruths = [1 if i=="s" else 0 for i in groundtruths]
        f1.append(f1_score(groundtruths, predictions, average='weighted'))
    # Print average ± standard deviation
    print("Accuracy " + str(sum(acc)/float(len(acc))) + " ± " + str(stdev(acc)))
    print("F-Measure " + str(sum(f1)/float(len(f1))) + " ± " + str(stdev(f1)))
    dir_stats = utils.create_dir("stats/")
    with open(dir_stats + "table1_accuracy.csv", "a") as filep:
        filep.write("SVMBFF")
        for val in acc:
            filep.write("," + str(val))
        filep.write("\n")
    with open(dir_stats + "table1_f1.csv", "a") as filep:
        filep.write("SVMBFF")
        for val in f1:
            filep.write("," + str(val))
        filep.write("\n")
    # with open(dir_stats + "table1_accuracy.csv", "a") as filep:
    #     for val in acc:
    #         filep.write("SVMBFF," + str(val) + "\n")
    # with open(dir_stats + "table1_f1.csv", "a") as filep:
    #     for val in f1:
    #         filep.write("SVMBFF," + str(val) + "\n")

def extract_features(tracks_dir="tracks/", feat_dir="features/"):
    utils.print_success("Extracting features")
    tracks_fn = os.listdir(tracks_dir)
    utils.create_dir(feat_dir)
    feat_dir = utils.create_dir(feat_dir + "svmbff")
    bextract = "bextract -mfcc -zcrs -ctd -rlf -flx -ws 1024 -as 898 -sv -fe "
    for index, filename in enumerate(tracks_fn):
        utils.print_progress_start(str(index) + "/" + str(len(tracks_fn)) + " " + filename)
        track_path = filename + ".mf"
        with open(track_path, "w") as filep:
            filep.write(tracks_dir + filename + "\n")
        new_fn = filename.split(".")[0] + ".arff"
        try:
            os.system(bextract + track_path + " -w " + new_fn + "> /dev/null 2>&1")
        except:
            utils.print_info("You have to make marsyas available systemwide, tips:")
            utils.print_info("http://marsyas.info/doc/manual/marsyas-user/Step_002dby_002dstep-building-instructions.html#Step_002dby_002dstep-building-instructions")
            utils.print_info("http://stackoverflow.com/a/21173918")
            utils.print_error("Program exit")
        # print(new_fn)
        # print(feat_dir + " " + new_fn)
        os.rename(new_fn, feat_dir + new_fn)
        # os.rename("MARSYAS_EMPTY" + new_fn, feat_dir + new_fn)
        os.system("rm " + track_path)
    utils.print_progress_end()
    os.system("rm bextract_single.mf")

def table1_exp1(folds_dir):
    utils.print_success("Experiment 1 in Table 1")
    fn_gts = "groundtruths/database1.csv"
    gts = utils.read_groundtruths(fn_gts)
    res_files = [name for name in os.listdir(folds_dir) if os.path.isfile(os.path.join(folds_dir, name)) and "results" in name]
    acc = []
    f1 = []
    for res in res_files:
        predictions = []
        groundtruths = []
        preds = read_preds(folds_dir + res)
        for name in preds:
            name_gts = name.split(".")[0]
            if name_gts in gts:
                groundtruths.append(gts[name_gts])
                predictions.append(preds[name])
        acc.append(accuracy_score(groundtruths, predictions))
        predictions = [1 if i=="s" else 0 for i in predictions]
        groundtruths = [1 if i=="s" else 0 for i in groundtruths]
        f1.append(f1_score(groundtruths, predictions, average='binary'))
    # Print average ± standard deviation
    utils.print_info("Accuracy " + str(sum(acc)/float(len(acc))) + " ± " + str(stdev(acc)))
    utils.print_info("F-Measure " + str(sum(f1)/float(len(f1))) + " ± " + str(stdev(f1)))
    dir_res = utils.create_dir("stats/")
    with open(dir_res + "table1_accuracy.csv", "a") as filep:
        for val in acc:
            filep.write("SVMBFF," + str(val) + "\n")
    with open(dir_res + "table1_f1.csv", "a") as filep:
        for val in f1:
            filep.write("SVMBFF," + str(val) + "\n")

def experiment_1(folder="."):
    utils.print_success("SVMBFF Experiment 1 (approx. 1 minutes)")

    # Variables
    folder = utils.abs_path_dir(folder)
    dir_tmp = utils.create_dir(folder + "src/tmp/")
    dir_svmbff = utils.create_dir(dir_tmp + "svmbff/")
    dir_tracks = folder + "tracks/"
    dir_feat = folder + "features/svmbff/"
    fn_feats_db1 = dir_svmbff + "svmbff_database1.arff"
    feats_gts_db1 = folder + "features/" + "svmbff_database1.arff"
    groundtruths = folder + "groundtruths/database1.csv"

    extract_features(dir_tracks)
    merge_arff(dir_feat, fn_feats_db1)
    add_groundtruth(fn_feats_db1, groundtruths, feats_gts_db1)
    os.remove(fn_feats_db1)
    dir_folds = create_folds(feats_gts_db1, 5, dir_svmbff)
    run_kea_on_folds(dir_folds)
    table1_exp1(dir_folds)

def main():
    utils.print_success("SVMBFF (approx. 2 minutes)")
    experiment_1(folder="../")

if __name__ == "__main__":
    PARSER = argparse.ArgumentParser(description="Validate list of ISRCs")
    PARSER.add_argument(
        "-i",
        "--input_dir",
        help="input directory containing all ARFF file from Marsyas bextract",
        type=str,
        default="data",
        metavar="input_dir")
    PARSER.add_argument(
        "-o",
        "--output_file",
        help="output file",
        type=str,
        default="feat_with_groundtruth.txt",
        metavar="output_file")
    PARSER.add_argument(
        "-g",
        "--groundtruth_file",
        help="groundtruth file",
        type=str,
        default="groundtruth.txt",
        metavar="groundtruth_file")
    PARSER.add_argument(
        "-n",
        "--nb_folds",
        default=1,
        type=int,
        metavar="nb_folds",
        help="classification folds number, must be >= 1, default = 1")

    main()
    # figure2()
    # indir1 = "res/"
    # indir2 = "/media/sf_DATA/Datasets/Simbals/new/201611/arff/"
    # merge_arff(indir2, "test2.arff")

    # utils.print_success("Kea classification")
    # # Variable declaration
    # input_dir = PARSER.parse_args().input_dir
    # res_dir = "analysis"
    # utils.create_dir(res_dir)
    # if input_dir[-1] == "/":
    #     input_dir = input_dir[:-1]
    # proj_dir = res_dir + "/" + input_dir.split("/")[-1]
    # utils.create_dir(proj_dir)
    # feat_without_groundtruth = proj_dir + "/feat_without_groundtruth.arff"
    # feat_with_groundtruth = proj_dir + "/" + PARSER.parse_args().output_file
    # # Functions call
    # merge_arff(input_dir, feat_without_groundtruth)
    # add_groundtruth(feat_without_groundtruth,
    #     PARSER.parse_args().groundtruth_file,
    #     feat_with_groundtruth)
    # os.remove(feat_without_groundtruth)
    # folds_dir = create_folds(feat_with_groundtruth, PARSER.parse_args().nb_folds, invert_train_test=True)
    # folds_dir = create_folds("results/train_kea.arff", 5)
    # run_kea_on_folds(folds_dir)

# # 2 merge all arff files dans train/test file (generate train/test folds/set,
# #   reuse vqmm) à partir des fichiers sources d'un autre dossier, tout copier
# #   dans dossier de svmbff. no-overlap train/Test
# # 3 lancer kea sur toutes les train/test
# # 4 Afficher les résultats

#     utils.print_success("Finished in " + str(int(round(time.time() * 1000)) - begin) + "ms")

# """
# kea -m tags -w ' + train_file + ' -tw ' + test_file + ' -pr ' + out_file
# """


# -*- coding: utf-8 -*-

import pandas as pd

# Segregated collection of methods that require the BlockManager internal data
# structure

class TestSeriesBlockInternals(object):

    def test_setitem_invalidates_datetime_index_freq(self):
        # GH#24096 altering a datetime64tz Series inplace invalidates the
        #  `freq` attribute on the underlying DatetimeIndex

        dti = pd.date_range('20130101', periods=3, tz='US/Eastern')
        ts = dti[1]
        ser = pd.Series(dti)
        assert ser._values is not dti
        assert ser._values._data.base is not dti._data._data.base
        assert dti.freq == 'D'
        ser.iloc[1] = pd.NaT
        assert ser._values.freq is None

        # check that the DatetimeIndex was not altered in place
        assert ser._values is not dti
        assert ser._values._data.base is not dti._data._data.base
        assert dti[1] == ts
        assert dti.freq == 'D'

    def test_dt64tz_setitem_does_not_mutate_dti(self):
        # GH#21907, GH#24096
        dti = pd.date_range('2016-01-01', periods=10, tz='US/Pacific')
        ts = dti[0]
        ser = pd.Series(dti)
        assert ser._values is not dti
        assert ser._values._data.base is not dti._data._data.base
        assert ser._data.blocks[0].values is not dti
        assert (ser._data.blocks[0].values._data.base
                is not dti._data._data.base)

        ser[::3] = pd.NaT
        assert ser[0] is pd.NaT
        assert dti[0] == ts


# -*- coding: utf-8 -*-
def sim_normal(nums, mean = 600, sd = 30):
    import numpy as np
    import numpy.random as nr
    for n in nums:
        dist = nr.normal(loc = mean, scale = sd, size = n)
        titl = 'Normal distribution with ' + str(n) + ' values'
        print('Summary for ' + str(n) + ' samples')
        print(dist_summary(dist, titl))
        print('Emperical 95% CIs')
        print(np.percentile(dist, [2.5, 97.5]))
        print(' ')
    return('Done!')

def sim_poisson(nums, mean = 600):
    import numpy as np
    import numpy.random as nr
    for n in nums:
        dist = nr.poisson(lam = mean, size = n)
        titl = 'Poisson distribution with ' + str(n) + ' values'
        print(dist_summary(dist, titl))
        print('Emperical 95% CIs')
        print(np.percentile(dist, [2.5, 97.5]))
        print(' ')
    return('Done!')

def dist_summary(dist, names = 'dist_name'):
    import pandas as pd
    import matplotlib.pyplot as plt
    ser = pd.Series(dist)
    fig = plt.figure(1, figsize=(9, 6))
    ax = fig.gca()
    ser.hist(ax = ax, bins = 120)
    ax.set_title('Frequency distribution of ' + names)
    ax.set_ylabel('Frequency')
    plt.show()
    return(ser.describe())

def gen_profits(num):
    import numpy.random as nr
    unif = nr.uniform(size = num)
    out = [5 if x < 0.3 else (3.5 if x < 0.6 else 4) for x in unif]
    return(out)

def gen_tips(num):
    import numpy.random as nr
    unif = nr.uniform(size = num)
    out = [0 if x < 0.5 else (0.25 if x < 0.7
      else (1.0 if x < 0.9 else 2.0)) for x in unif]
    return(out)

def sim_lemonade(num, mean = 600, sd = 30, pois = False):
  ## Simulate the profits and tips for
  ## a lemonade stand.
  import numpy.random as nr

  ## number of customer arrivals
  if pois:
    arrivals = nr.poisson(lam = mean, size = num)
  else:
   arrivals = nr.normal(loc = mean, scale = sd, size = num)

  print(dist_summary(arrivals, 'customer arrivals per day'))

  ## Compute distibution of average profit per arrival
  proft = gen_profits(num)
  print(dist_summary(proft, 'profit per arrival'))

  ## Total profits are profit per arrival
  ## times number of arrivals.
  total_profit = arrivals * proft
  print(dist_summary(total_profit, 'total profit per day'))

  ## Compute distribution of average tips per arrival
  tps = gen_tips(num)
  print(dist_summary(tps, 'tips per arrival'))

  ## Compute average tips per day
  total_tips = arrivals * tps
  print(dist_summary(total_tips, 'total tips per day'))

  ## Compute total profits plus total tips.
  total_take = total_profit + total_tips
  return(dist_summary(total_take, 'total net per day'))


import re, inspect, textwrap, pydoc
import sphinx
from docscrape import NumpyDocString, FunctionDoc, ClassDoc

class SphinxDocString(NumpyDocString):
    def __init__(self, docstring, config={}):
        self.use_plots = config.get('use_plots', False)
        NumpyDocString.__init__(self, docstring, config=config)

    # string conversion routines
    def _str_header(self, name, symbol='`'):
        return ['.. rubric:: ' + name, '']

    def _str_field_list(self, name):
        return [':' + name + ':']

    def _str_indent(self, doc, indent=4):
        out = []
        for line in doc:
            out += [' '*indent + line]
        return out

    def _str_signature(self):
        return ['']
        if self['Signature']:
            return ['``%s``' % self['Signature']] + ['']
        else:
            return ['']

    def _str_summary(self):
        return self['Summary'] + ['']

    def _str_extended_summary(self):
        return self['Extended Summary'] + ['']

    def _str_param_list(self, name):
        out = []
        if self[name]:
            out += self._str_field_list(name)
            out += ['']
            for param,param_type,desc in self[name]:
                out += self._str_indent(['**%s** : %s' % (param.strip(),
                                                          param_type)])
                out += ['']
                out += self._str_indent(desc,8)
                out += ['']
        return out

    @property
    def _obj(self):
        if hasattr(self, '_cls'):
            return self._cls
        elif hasattr(self, '_f'):
            return self._f
        return None

    def _str_member_list(self, name):
        """
        Generate a member listing, autosummary:: table where possible,
        and a table where not.

        """
        out = []
        if self[name]:
            out += ['.. rubric:: %s' % name, '']
            prefix = getattr(self, '_name', '')

            if prefix:
                prefix = '~%s.' % prefix

            autosum = []
            others = []
            for param, param_type, desc in self[name]:
                param = param.strip()
                if not self._obj or hasattr(self._obj, param):
                    autosum += ["   %s%s" % (prefix, param)]
                else:
                    others.append((param, param_type, desc))

            if autosum:
                out += ['.. autosummary::', '   :toctree:', '']
                out += autosum

            if others:
                maxlen_0 = max([len(x[0]) for x in others])
                maxlen_1 = max([len(x[1]) for x in others])
                hdr = "="*maxlen_0 + "  " + "="*maxlen_1 + "  " + "="*10
                fmt = '%%%ds  %%%ds  ' % (maxlen_0, maxlen_1)
                n_indent = maxlen_0 + maxlen_1 + 4
                out += [hdr]
                for param, param_type, desc in others:
                    out += [fmt % (param.strip(), param_type)]
                    out += self._str_indent(desc, n_indent)
                out += [hdr]
            out += ['']
        return out

    def _str_section(self, name):
        out = []
        if self[name]:
            out += self._str_header(name)
            out += ['']
            content = textwrap.dedent("\n".join(self[name])).split("\n")
            out += content
            out += ['']
        return out

    def _str_see_also(self, func_role):
        out = []
        if self['See Also']:
            see_also = super(SphinxDocString, self)._str_see_also(func_role)
            out = ['.. seealso::', '']
            out += self._str_indent(see_also[2:])
        return out

    def _str_warnings(self):
        out = []
        if self['Warnings']:
            out = ['.. warning::', '']
            out += self._str_indent(self['Warnings'])
        return out

    def _str_index(self):
        idx = self['index']
        out = []
        if len(idx) == 0:
            return out

        out += ['.. index:: %s' % idx.get('default','')]
        for section, references in idx.iteritems():
            if section == 'default':
                continue
            elif section == 'refguide':
                out += ['   single: %s' % (', '.join(references))]
            else:
                out += ['   %s: %s' % (section, ','.join(references))]
        return out

    def _str_references(self):
        out = []
        if self['References']:
            out += self._str_header('References')
            if isinstance(self['References'], str):
                self['References'] = [self['References']]
            out.extend(self['References'])
            out += ['']
            # Latex collects all references to a separate bibliography,
            # so we need to insert links to it
            if sphinx.__version__ >= "0.6":
                out += ['.. only:: latex','']
            else:
                out += ['.. latexonly::','']
            items = []
            for line in self['References']:
                m = re.match(r'.. \[([a-z0-9._-]+)\]', line, re.I)
                if m:
                    items.append(m.group(1))
            out += ['   ' + ", ".join(["[%s]_" % item for item in items]), '']
        return out

    def _str_examples(self):
        examples_str = "\n".join(self['Examples'])

        if (self.use_plots and 'import matplotlib' in examples_str
                and 'plot::' not in examples_str):
            out = []
            out += self._str_header('Examples')
            out += ['.. plot::', '']
            out += self._str_indent(self['Examples'])
            out += ['']
            return out
        else:
            return self._str_section('Examples')

    def __str__(self, indent=0, func_role="obj"):
        out = []
        out += self._str_signature()
        out += self._str_index() + ['']
        out += self._str_summary()
        out += self._str_extended_summary()
        for param_list in ('Parameters', 'Returns', 'Raises'):
            out += self._str_param_list(param_list)
        out += self._str_warnings()
        out += self._str_see_also(func_role)
        out += self._str_section('Notes')
        out += self._str_references()
        out += self._str_examples()
        for param_list in ('Attributes', 'Methods'):
            out += self._str_member_list(param_list)
        out = self._str_indent(out,indent)
        return '\n'.join(out)

class SphinxFunctionDoc(SphinxDocString, FunctionDoc):
    def __init__(self, obj, doc=None, config={}):
        self.use_plots = config.get('use_plots', False)
        FunctionDoc.__init__(self, obj, doc=doc, config=config)

class SphinxClassDoc(SphinxDocString, ClassDoc):
    def __init__(self, obj, doc=None, func_doc=None, config={}):
        self.use_plots = config.get('use_plots', False)
        ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config)

class SphinxObjDoc(SphinxDocString):
    def __init__(self, obj, doc=None, config={}):
        self._f = obj
        SphinxDocString.__init__(self, doc, config=config)

def get_doc_object(obj, what=None, doc=None, config={}):
    if what is None:
        if inspect.isclass(obj):
            what = 'class'
        elif inspect.ismodule(obj):
            what = 'module'
        elif callable(obj):
            what = 'function'
        else:
            what = 'object'
    if what == 'class':
        return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc,
                              config=config)
    elif what in ('function', 'method'):
        return SphinxFunctionDoc(obj, doc=doc, config=config)
    else:
        if doc is None:
            doc = pydoc.getdoc(obj)
        return SphinxObjDoc(obj, doc, config=config)


"""
Usage: python_visitor_gui.py

This script shows how one can implement visitors
in pure python and inject them into OpenGM solver.
( not all OpenGM solvers support this kind of
    code injection )
"""

import opengm
import numpy
import matplotlib
from matplotlib import pyplot as plt

shape=[100,100]
numLabels=10
unaries=numpy.random.rand(shape[0], shape[1],numLabels)
potts=opengm.PottsFunction([numLabels,numLabels],0.0,0.4)
gm=opengm.grid2d2Order(unaries=unaries,regularizer=potts)
inf=opengm.inference.BeliefPropagation(gm,parameter=opengm.InfParam(damping=0.5))

class PyCallback(object):
    def __init__(self,shape,numLabels):
        self.shape=shape
        self.numLabels=numLabels
        self.cmap = matplotlib.colors.ListedColormap ( numpy.random.rand ( self.numLabels,3))
        matplotlib.interactive(True)
    def begin(self,inference):
        print "begin of inference"
    def end(self,inference):
        print "end of inference"
    def visit(self,inference):
        gm=inference.gm()
        labelVector=inference.arg()
        print "energy  ",gm.evaluate(labelVector)
        labelVector=labelVector.reshape(self.shape)
        plt.imshow(labelVector*255.0, cmap=self.cmap,interpolation="nearest")
        plt.draw()

callback=PyCallback(shape,numLabels)
visitor=inf.pythonVisitor(callback,visitNth=1)

inf.infer(visitor)
argmin=inf.arg()


from .config import *
from ..utils import rebin, db
from .plot_utils import calc_extent

def plot_time_series(wf, f_start=None, f_stop=None, if_id=0, logged=True, orientation='h', MJD_time=False, **kwargs):
    """ Plot the time series.

     Args:
        f_start (float): start frequency, in MHz
        f_stop (float): stop frequency, in MHz
        logged (bool): Plot in linear (False) or dB units (True),
        kwargs: keyword args to be passed to matplotlib imshow()
    """

    ax = plt.gca()
    plot_f, plot_data = wf.grab_data(f_start, f_stop, if_id)

    # Since the data has been squeezed, the axis for time goes away if only one bin, causing a bug with axis=1
    if len(plot_data.shape) > 1:
        plot_data = np.nanmean(plot_data, axis=1)
    else:
        plot_data = np.nanmean(plot_data)

    if logged and wf.header['nbits'] >= 8:
        plot_data = db(plot_data)

    # Make proper time axis for plotting (but only for plotting!). Note that this makes the values inclusive.
    extent = calc_extent(wf, plot_f=plot_f, plot_t=wf.timestamps, MJD_time=MJD_time)
    plot_t = np.linspace(extent[2], extent[3], len(wf.timestamps))

    if MJD_time:
        tlabel = "Time [MJD]"
    else:
        tlabel = "Time [s]"

    if logged:
        plabel = "Power [dB]"
    else:
        plabel = "Power [counts]"

    # Reverse oder if vertical orientation.
    if 'v' in orientation:
        plt.plot(plot_data, plot_t, **kwargs)
        plt.xlabel(plabel)

    else:
        plt.plot(plot_t, plot_data, **kwargs)
        plt.xlabel(tlabel)
        plt.ylabel(plabel)

    ax.autoscale(axis='both', tight=True)


import numpy as np
import pandas as pd
import scipy.io as sio

from my_settings import *

data = sio.loadmat("/home/mje/Projects/agency_connectivity/Data/data_all.mat")[
    "data_all"]

column_keys = ["subject", "trial", "condition", "shift"]
result_df = pd.DataFrame(columns=column_keys)

for k, subject in enumerate(subjects):

    p8_invol_shift = data[k, 3] - np.mean(data[k, 0])
    p8_vol_shift = data[k, 2] - np.mean(data[k, 0])
    p8_vol_bs_shift = data[k, 1] - np.mean(data[k, 0])

    for j in range(89):
        row = pd.DataFrame([{"trial": int(j),
                             "subject": subject,
                             "condition": "vol_bs",
                             "shift": p8_vol_bs_shift[j + 1][0]}])

        result_df = result_df.append(row, ignore_index=True)

    for j in range(89):
        row = pd.DataFrame([{"trial": int(j),
                             "subject": subject,
                             "condition": "vol",
                             "shift": p8_vol_shift[j + 1][0]}])

        result_df = result_df.append(row, ignore_index=True)

    for j in range(89):
        row = pd.DataFrame([{"trial": int(j),
                             "subject": subject,
                             "condition": "invol",
                             "shift": p8_invol_shift[j][0]}])

        result_df = result_df.append(row, ignore_index=True)


# -*- coding: utf-8 -*-

# Copyright 2015 Dominik Fay
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from bisect import bisect_left, bisect_right
import sys
import re
from pathlib import Path

from warnings import warn
import numpy as np

from pyimzml.metadata import Metadata, SpectrumData
from pyimzml.ontology.ontology import convert_cv_param

PRECISION_DICT = {"32-bit float": 'f', "64-bit float": 'd', "32-bit integer": 'i', "64-bit integer": 'l'}
SIZE_DICT = {'f': 4, 'd': 8, 'i': 4, 'l': 8}
INFER_IBD_FROM_IMZML = object()
XMLNS_PREFIX = "{http://psi.hupo.org/ms/mzml}"

param_group_elname = "referenceableParamGroup"
data_processing_elname = "dataProcessing"
instrument_confid_elname = "instrumentConfiguration"

def choose_iterparse(parse_lib=None):
    if parse_lib == 'ElementTree':
        from xml.etree.ElementTree import iterparse
    elif parse_lib == 'lxml':
        from lxml.etree import iterparse
    else:
        try:
            from lxml.etree import iterparse
        except ImportError:
            from xml.etree.ElementTree import iterparse
    return iterparse

def _get_cv_param(elem, accession, deep=False, convert=False):
    base = './/' if deep else ''
    node = elem.find('%s%scvParam[@accession="%s"]' % (base, XMLNS_PREFIX, accession))
    if node is not None:
        if convert:
            return convert_cv_param(accession, node.get('value'))
        return node.get('value')

class ImzMLParser:
    """
    Parser for imzML 1.1.0 files (see specification here:
    http://imzml.org/download/imzml/specifications_imzML1.1.0_RC1.pdf).

    Iteratively reads the .imzML file into memory while pruning the per-spectrum metadata (everything in
    <spectrumList> elements) during initialization. Returns a spectrum upon calling getspectrum(i). The binary file
    is read in every call of getspectrum(i). Use enumerate(parser.coordinates) to get all coordinates with their
    respective index. Coordinates are always 3-dimensional. If the third spatial dimension is not present in
    the data, it will be set to zero.

    The global metadata fields in the imzML file are stored in parser.metadata.
    Spectrum-specific metadata fields are not stored by default due to avoid memory issues,
    use the `include_spectra_metadata` parameter if spectrum-specific metadata is needed.
    """

    def __init__(
            self,
            filename,
            parse_lib=None,
            ibd_file=INFER_IBD_FROM_IMZML,
            include_spectra_metadata=None,
    ):
        """
        Opens the two files corresponding to the file name, reads the entire .imzML
        file and extracts required attributes. Does not read any binary data, yet.

        :param filename:
            name of the XML file. Must end with .imzML. Binary data file must be named equally but ending with .ibd
            Alternatively an open file or Buffer Protocol object can be supplied, if ibd_file is also supplied
        :param parse_lib:
            XML-parsing library to use: 'ElementTree' or 'lxml', the later will be used if argument not provided
        :param ibd_file:
            File or Buffer Protocol object for the .ibd file. Leave blank to infer it from the imzml filename.
            Set to None if no data from the .ibd file is needed (getspectrum calls will not work)
        :param include_spectra_metadata:
            None, 'full', or a list/set of accession IDs.
            If 'full' is given, parser.spectrum_full_metadata will be populated with a list of
                complex objects containing the full metadata for each spectrum.
            If a list or set is given, parser.spectrum_metadata_fields will be populated with a dict mapping
                accession IDs to lists. Each list will contain the values for that accession ID for
                each spectrum. Note that for performance reasons, this mode only searches the
                spectrum itself for the value. It won't check any referenced referenceable param
                groups if the accession ID isn't present in the spectrum metadata.
        """
        # ElementTree requires the schema location for finding tags (why?) but
        # fails to read it from the root element. As this should be identical
        # for all imzML files, it is hard-coded here and prepended before every tag
        self.sl = "{http://psi.hupo.org/ms/mzml}"
        # maps each imzML number format to its struct equivalent
        self.precisionDict = dict(PRECISION_DICT)
        # maps each number format character to its amount of bytes used
        self.sizeDict = dict(SIZE_DICT)
        self.filename = filename
        self.mzOffsets = []
        self.intensityOffsets = []
        self.mzLengths = []
        self.intensityLengths = []
        # list of all (x,y,z) coordinates as tuples.
        self.coordinates = []
        self.root = None
        self.metadata = None
        if include_spectra_metadata == 'full':
            self.spectrum_full_metadata = []
        elif include_spectra_metadata is not None:
            include_spectra_metadata = set(include_spectra_metadata)
            self.spectrum_metadata_fields = {
                k: [] for k in include_spectra_metadata
            }

        self.mzGroupId = self.intGroupId = self.mzPrecision = self.intensityPrecision = None
        self.iterparse = choose_iterparse(parse_lib)
        self.__iter_read_spectrum_meta(include_spectra_metadata)
        if ibd_file is INFER_IBD_FROM_IMZML:
            # name of the binary file
            ibd_filename = self._infer_bin_filename(self.filename)
            self.m = open(ibd_filename, "rb")
        else:
            self.m = ibd_file

        # Dict for basic imzML metadata other than those required for reading
        # spectra. See method __readimzmlmeta()
        self.imzmldict = self.__readimzmlmeta()
        self.imzmldict['max count of pixels z'] = np.asarray(self.coordinates)[:,2].max()

    @staticmethod
    def _infer_bin_filename(imzml_path):
        imzml_path = Path(imzml_path)
        ibd_path = [f for f in imzml_path.parent.glob('*')
                    if re.match(r'.+\.ibd', str(f), re.IGNORECASE) and f.stem == imzml_path.stem][0]
        return str(ibd_path)

    # system method for use of 'with ... as'
    def __enter__(self):
        return self

    # system method for use of 'with ... as'
    def __exit__(self, exc_t, exc_v, trace):
        if self.m is not None:
            self.m.close()

    def __iter_read_spectrum_meta(self, include_spectra_metadata):
        """
        This method should only be called by __init__. Reads the data formats, coordinates and offsets from
        the .imzML file and initializes the respective attributes. While traversing the XML tree, the per-spectrum
        metadata is pruned, i.e. the <spectrumList> element(s) are left behind empty.

        Supported accession values for the number formats: "MS:1000521", "MS:1000523", "IMS:1000141" or
        "IMS:1000142". The string values are "32-bit float", "64-bit float", "32-bit integer", "64-bit integer".
        """
        mz_group = int_group = None
        slist = None
        elem_iterator = self.iterparse(self.filename, events=("start", "end"))

        if sys.version_info > (3,):
            _, self.root = next(elem_iterator)
        else:
            _, self.root = elem_iterator.next()

        for event, elem in elem_iterator:
            if elem.tag == self.sl + "spectrumList" and event == "start":
                self.__process_metadata()
                slist = elem
            elif elem.tag == self.sl + "spectrum" and event == "end":
                self.__process_spectrum(elem, include_spectra_metadata)
                slist.remove(elem)
        self.__fix_offsets()

    def __fix_offsets(self):
        # clean up the mess after morons who use signed 32-bit where unsigned 64-bit is appropriate
        def fix(array):
            fixed = []
            delta = 0
            prev_value = float('nan')
            for value in array:
                if value < 0 and prev_value >= 0:
                    delta += 2**32
                fixed.append(value + delta)
                prev_value = value
            return fixed

        self.mzOffsets = fix(self.mzOffsets)
        self.intensityOffsets = fix(self.intensityOffsets)

    def __process_metadata(self):
        if self.metadata is None:
            self.metadata = Metadata(self.root)
            for param_id, param_group in self.metadata.referenceable_param_groups.items():
                if 'm/z array' in param_group.param_by_name:
                    self.mzGroupId = param_id
                    for name, dtype in self.precisionDict.items():
                        if name in param_group.param_by_name:
                            self.mzPrecision = dtype
                if 'intensity array' in param_group.param_by_name:
                    self.intGroupId = param_id
                    for name, dtype in self.precisionDict.items():
                        if name in param_group.param_by_name:
                            self.intensityPrecision = dtype
            if not hasattr(self, 'mzPrecision'):
                raise RuntimeError("Could not determine m/z precision")
            if not hasattr(self, 'intensityPrecision'):
                raise RuntimeError("Could not determine intensity precision")

    def __process_spectrum(self, elem, include_spectra_metadata):
        arrlistelem = elem.find('%sbinaryDataArrayList' % self.sl)
        mz_group = None
        int_group = None
        for e in arrlistelem:
            ref = e.find('%sreferenceableParamGroupRef' % self.sl).attrib["ref"]
            if ref == self.mzGroupId:
                mz_group = e
            elif ref == self.intGroupId:
                int_group = e
        self.mzOffsets.append(int(_get_cv_param(mz_group, 'IMS:1000102')))
        self.mzLengths.append(int(_get_cv_param(mz_group, 'IMS:1000103')))
        self.intensityOffsets.append(int(_get_cv_param(int_group, 'IMS:1000102')))
        self.intensityLengths.append(int(_get_cv_param(int_group, 'IMS:1000103')))
        scan_elem = elem.find('%sscanList/%sscan' % (self.sl, self.sl))
        x = _get_cv_param(scan_elem, 'IMS:1000050')
        y = _get_cv_param(scan_elem, 'IMS:1000051')
        z = _get_cv_param(scan_elem, 'IMS:1000052')
        if z is not None:
            self.coordinates.append((int(x), int(y), int(z)))
        else:
            self.coordinates.append((int(x), int(y), 1))

        if include_spectra_metadata == 'full':
            self.spectrum_full_metadata.append(
                SpectrumData(elem, self.metadata.referenceable_param_groups)
            )
        elif include_spectra_metadata:
            for param in include_spectra_metadata:
                value = _get_cv_param(elem, param, deep=True, convert=True)
                self.spectrum_metadata_fields[param].append(value)

    def __readimzmlmeta(self):
        """
        DEPRECATED - use self.metadata instead, as it has much greater detail and allows for
        multiple scan settings / instruments.

        This method should only be called by __init__. Initializes the imzmldict with frequently used metadata from
        the .imzML file.

        :return d:
            dict containing above mentioned meta data
        :rtype:
            dict
        :raises Warning:
            if an xml attribute has a number format different from the imzML specification
        """
        d = {}
        scan_settings_list_elem = self.root.find('%sscanSettingsList' % self.sl)
        instrument_config_list_elem = self.root.find('%sinstrumentConfigurationList' % self.sl)
        scan_settings_params = [
            ("max count of pixels x", "IMS:1000042"),
            ("max count of pixels y", "IMS:1000043"),
            ("max dimension x", "IMS:1000044"),
            ("max dimension y", "IMS:1000045"),
            ("pixel size x", "IMS:1000046"),
            ("pixel size y", "IMS:1000047"),
            ("matrix solution concentration", "MS:1000835"),
        ]
        instrument_config_params = [
            ("wavelength", "MS:1000843"),
            ("focus diameter x", "MS:1000844"),
            ("focus diameter y", "MS:1000845"),
            ("pulse energy", "MS:1000846"),
            ("pulse duration", "MS:1000847"),
            ("attenuation", "MS:1000848"),
        ]

        for name, accession in scan_settings_params:
            try:
                val = _get_cv_param(scan_settings_list_elem, accession, deep=True, convert=True)
                if val is not None:
                    d[name] = val
            except ValueError:
                warn(Warning('Wrong data type in XML file. Skipped attribute "%s"' % name))

        for name, accession in instrument_config_params:
            try:
                val = _get_cv_param(instrument_config_list_elem, accession, deep=True, convert=True)
                if val is not None:
                    d[name] = val
            except ValueError:
                warn(Warning('Wrong data type in XML file. Skipped attribute "%s"' % name))
        return d

    def get_physical_coordinates(self, i):
        """
        For a pixel index i, return the real-world coordinates in nanometers.

        This is equivalent to multiplying the image coordinates of the given pixel with the pixel size.

        :param i: the pixel index
        :return: a tuple of x and y coordinates.
        :rtype: Tuple[float]
        :raises KeyError: if the .imzML file does not specify the attributes "pixel size x" and "pixel size y"
        """
        try:
            pixel_size_x = self.imzmldict["pixel size x"]
            pixel_size_y = self.imzmldict["pixel size y"]
        except KeyError:
            raise KeyError("Could not find all pixel size attributes in imzML file")
        image_x, image_y = self.coordinates[i][:2]
        return image_x * pixel_size_x, image_y * pixel_size_y

    def getspectrum(self, index):
        """
        Reads the spectrum at specified index from the .ibd file.

        :param index:
            Index of the desired spectrum in the .imzML file

        Output:

        mz_array: numpy.ndarray
            Sequence of m/z values representing the horizontal axis of the desired mass
            spectrum
        intensity_array: numpy.ndarray
            Sequence of intensity values corresponding to mz_array
        """
        mz_bytes, intensity_bytes = self.get_spectrum_as_string(index)
        mz_array = np.frombuffer(mz_bytes, dtype=self.mzPrecision)
        intensity_array = np.frombuffer(intensity_bytes, dtype=self.intensityPrecision)
        return mz_array, intensity_array

    def get_spectrum_as_string(self, index):
        """
        Reads m/z array and intensity array of the spectrum at specified location
        from the binary file as a byte string. The string can be unpacked by the struct
        module. To get the arrays as numbers, use getspectrum

        :param index:
            Index of the desired spectrum in the .imzML file
        :rtype: Tuple[str, str]

        Output:

        mz_string:
            string where each character represents a byte of the mz array of the
            spectrum
        intensity_string:
            string where each character represents a byte of the intensity array of
            the spectrum
        """
        offsets = [self.mzOffsets[index], self.intensityOffsets[index]]
        lengths = [self.mzLengths[index], self.intensityLengths[index]]
        lengths[0] *= self.sizeDict[self.mzPrecision]
        lengths[1] *= self.sizeDict[self.intensityPrecision]
        self.m.seek(offsets[0])
        mz_string = self.m.read(lengths[0])
        self.m.seek(offsets[1])
        intensity_string = self.m.read(lengths[1])
        return mz_string, intensity_string

    def portable_spectrum_reader(self):
        """
        Builds a PortableSpectrumReader that holds the coordinates list and spectrum offsets in the .ibd file
        so that the .ibd file can be read without opening the .imzML file again.

        The PortableSpectrumReader can be safely pickled and unpickled, making it useful for reading the spectra
        in a distributed environment such as PySpark or PyWren.
        """
        return PortableSpectrumReader(self.coordinates,
                                      self.mzPrecision, self.mzOffsets, self.mzLengths,
                                      self.intensityPrecision, self.intensityOffsets, self.intensityLengths)

def getionimage(p, mz_value, tol=0.1, z=1, reduce_func=sum):
    """
    Get an image representation of the intensity distribution
    of the ion with specified m/z value.

    By default, the intensity values within the tolerance region are summed.

    :param p:
        the ImzMLParser (or anything else with similar attributes) for the desired dataset
    :param mz_value:
        m/z value for which the ion image shall be returned
    :param tol:
        Absolute tolerance for the m/z value, such that all ions with values
        mz_value-|tol| <= x <= mz_value+|tol| are included. Defaults to 0.1
    :param z:
        z Value if spectrogram is 3-dimensional.
    :param reduce_func:
        the bahaviour for reducing the intensities between mz_value-|tol| and mz_value+|tol| to a single value. Must
        be a function that takes a sequence as input and outputs a number. By default, the values are summed.

    :return:
        numpy matrix with each element representing the ion intensity in this
        pixel. Can be easily plotted with matplotlib
    """
    tol = abs(tol)
    im = np.zeros((p.imzmldict["max count of pixels y"], p.imzmldict["max count of pixels x"]))
    for i, (x, y, z_) in enumerate(p.coordinates):
        if z_ == 0:
            UserWarning("z coordinate = 0 present, if you're getting blank images set getionimage(.., .., z=0)")
        if z_ == z:
            mzs, ints = map(lambda x: np.asarray(x), p.getspectrum(i))
            min_i, max_i = _bisect_spectrum(mzs, mz_value, tol)
            im[y - 1, x - 1] = reduce_func(ints[min_i:max_i+1])
    return im

def browse(p):
    """
    Create a per-spectrum metadata browser for the parser.
    Usage::

        # get a list of the instrument configurations used in the first pixel
        instrument_configurations = browse(p).for_spectrum(0).get_ids("instrumentConfiguration")

    Currently, ``instrumentConfiguration``, ``dataProcessing`` and ``referenceableParamGroup`` are supported.

    For browsing all spectra iteratively, you should by all means use **ascending** indices. Doing otherwise can result
    in quadratic runtime. The following example shows how to retrieve all unique instrumentConfigurations used::

        browser = browse(p)
        all_config_ids = set()
        for i, _ in enumerate(p.coordinates):
            all_config_ids.update(browser.for_spectrum(i).get_ids("instrumentConfiguration"))

    This is a list of ids with which you can find the corresponding ``<instrumentConfiguration>`` tag in the xml tree.

    :param p: the parser
    :return: the browser
    """
    return _ImzMLMetaDataBrowser(p.root, p.filename, p.sl)

def _bisect_spectrum(mzs, mz_value, tol):
    ix_l, ix_u = bisect_left(mzs, mz_value - tol), bisect_right(mzs, mz_value + tol) - 1
    if ix_l == len(mzs):
        return len(mzs), len(mzs)
    if ix_u < 1:
        return 0, 0
    if ix_u == len(mzs):
        ix_u -= 1
    if mzs[ix_l] < (mz_value - tol):
        ix_l += 1
    if mzs[ix_u] > (mz_value + tol):
        ix_u -= 1
    return ix_l, ix_u

class _ImzMLMetaDataBrowser(object):
    def __init__(self, root, fn, sl):
        self._root = root
        self._sl = sl
        self._fn = fn
        self._iter, self._previous, self._list_elem = None, None, None
        self.iterparse = choose_iterparse()

    def for_spectrum(self, i):
        if self._previous is None or i <= self._previous:
            self._iter = self.iterparse(self._fn, events=("start", "end"))
        for event, s in self._iter:
            if s.tag == self._sl + "spectrumList" and event == "start":
                self._list_elem = s
            elif s.tag == self._sl + "spectrum" and event == "end":
                self._list_elem.remove(s)
                if s.attrib["index"] == str(i):
                    self._previous = i
                    return _SpectrumMetaDataBrowser(self._root, self._sl, s)

class _SpectrumMetaDataBrowser(object):
    def __init__(self, root, sl, spectrum):
        self._root = root
        self._sl = sl
        self._spectrum = spectrum

    def get_ids(self, element):
        param_methods = {
            param_group_elname: self._find_referenceable_param_groups,
            data_processing_elname: self._find_data_processing,
            instrument_confid_elname: self._find_instrument_configurations,
        }
        try:
            return param_methods[element]()
        except KeyError as e:
            raise ValueError("Unsupported element: " + str(element))

    def _find_referenceable_param_groups(self):
        param_group_refs = self._spectrum.findall("%sreferenceableParamGroupRef" % self._sl)
        ids = map(lambda g: g.attrib["ref"], param_group_refs)
        return ids

    def _find_instrument_configurations(self):
        ids = None
        scan_list = self._spectrum.find("%sscanList" % self._sl)
        if scan_list:
            scans = scan_list.findall("%sscan[@instrumentConfigurationRef]" % self._sl)
            ids = map(lambda s: s.attrib["instrumentConfigurationRef"], scans)
        if not ids:
            run = self._root.find("%srun")
            try:
                return [run.attrib["defaultInstrumentConfigurationRef"]]
            except KeyError as _:
                return list()
        else:
            return ids

    def _find_data_processing(self):
        try:
            return self._spectrum.attrib["dataProcessingRef"]
        except KeyError as _:
            spectrum_list = self._root.find("%srun/%sspectrumList" % tuple(2 * [self._sl]))
            try:
                return [spectrum_list.attrib["defaultDataProcessingRef"]]
            except KeyError as _:
                return []

class PortableSpectrumReader(object):
    """
    A pickle-able class for holding the minimal set of data required for reading,
    without holding any references to open files that wouldn't survive pickling.
    """

    def __init__(self, coordinates, mzPrecision, mzOffsets, mzLengths,
                 intensityPrecision, intensityOffsets, intensityLengths):
        self.coordinates = coordinates
        self.mzPrecision = mzPrecision
        self.mzOffsets = mzOffsets
        self.mzLengths = mzLengths
        self.intensityPrecision = intensityPrecision
        self.intensityOffsets = intensityOffsets
        self.intensityLengths = intensityLengths

    def read_spectrum_from_file(self, file, index):
        """
        Reads the spectrum at specified index from the .ibd file.

        :param file:
            File or file-like object for the .ibd file
        :param index:
            Index of the desired spectrum in the .imzML file

        Output:

        mz_array: numpy.ndarray
            Sequence of m/z values representing the horizontal axis of the desired mass
            spectrum
        intensity_array: numpy.ndarray
            Sequence of intensity values corresponding to mz_array
        """
        file.seek(self.mzOffsets[index])
        mz_bytes = file.read(self.mzLengths[index] * SIZE_DICT[self.mzPrecision])
        file.seek(self.intensityOffsets[index])
        intensity_bytes = file.read(self.intensityLengths[index] * SIZE_DICT[self.intensityPrecision])

        mz_array = np.frombuffer(mz_bytes, dtype=self.mzPrecision)
        intensity_array = np.frombuffer(intensity_bytes, dtype=self.intensityPrecision)

        return mz_array, intensity_array


from __future__ import print_function
import numpy as np
from grid import standard_grid, negative_grid
from iterative_policy_evaluation import print_values, print_policy
import matplotlib.pyplot as plt
from monte_carlo_exploring_starts import max_dict

EPS = 1e-4
GAMMA = 0.9
ALL_POSSIBLE_ACTIONS = {'U', 'D', 'L', 'R'}

def random_action(a, eps=0.1):
    p = np.random.random()
    if(p < 1 - eps):
        return a
    else:
        return np.random.choice(list(ALL_POSSIBLE_ACTIONS))

# monte carlo sampling - finding out optimal policy (policy iteration)
def play_game(grid, policy):
    all_states = list(grid.actions.keys())
    state = (2, 0)
    # instead of taking random action at first step, consider the action which is probabilistic with the policy
    a = random_action(policy[state])

    grid.set_state(state)
    states_actions_rewards = [(state, a, 0)] # action is corresponding to the one which is going to be taken

    while True:
        r = grid.move(a)
        state = grid.current_state()
        #print(prev_state)

        # if game over, break the loop
        if  grid.game_over():
            states_actions_rewards.append((state, None, r)) # agent has hit the wall and we should not allow it to happen
            break
        else:
            # collect the next action that we are gonna take and insert into the trace
            a = random_action(policy[state])
            states_actions_rewards.append((state, a, r))

    # calculate the returns by working backwards from terminal state
    G = 0
    states_actions_returns = []

    for i, state_action_reward in enumerate(reversed(states_actions_rewards)):
        state, action, reward = state_action_reward
        if i != 0:
            states_actions_returns.append((state, action, G))
        G = reward + GAMMA * G

    states_actions_returns.reverse()
    return states_actions_returns

def max_dict(hash):
    max_key = None
    max_val = float('-inf')
    for k in hash:
        if(hash[k] > max_val):
            max_key, max_val = k, hash[k]

    return max_key, max_val

if __name__ == '__main__':
    #grid = standard_grid()
    grid = negative_grid(-0.1)
    print('grid')
    print_values(grid.rewards, grid)

    # init random policy
    policy = {}
    for s in grid.actions:
        policy[s] = np.random.choice(list(ALL_POSSIBLE_ACTIONS))

    print('policy')
    print_policy(policy, grid)

    # initialioze Q(s, a)
    Q = {}
    returns = {} # buffer to hold all the returns for a state during monte-carlo game plays
    for s in grid.actions: # if state is non terminal
        Q[s] = {}
        for a in ALL_POSSIBLE_ACTIONS:
            # for all the possible actions, initialize Q(s,a)
            Q[s][a] = 0
            returns[(s, a)] = []

    # deltas
    deltas = []
    for sample in range(5000):
        if sample % 500 == 0:
            print(sample)

        biggest_change = 0

        # generate an episode and adapt Q(s, a)
        states_actions_returns = play_game(grid, policy)
        seen_states_actions = set()

        for s, a, G in states_actions_returns:
            key = (s, a)
            if s not in seen_states_actions:
                old_q = Q[s][a]
                returns[key].append(G)
                Q[s][a] = np.mean(returns[key])
                seen_states_actions.add(key)
                biggest_change = max(biggest_change, abs(G - old_q))

        deltas.append(biggest_change)

        # policy improvement
        for s in Q:
            policy[s] = max_dict(Q[s])[0]

    plt.plot(deltas)
    plt.show()

    V = {}
    # policy improvement
    for s in Q:
        V[s] = max_dict(Q[s])[1]

    print('grid')
    print_values(V, grid)
    print('policy')
    print_policy(policy, grid)


from sklearn.covariance import EllipticEnvelope
import sklearn.neighbors
from sklearn.svm import OneClassSVM
import os
from msmbuilder import example_datasets, cluster, msm, featurizer, lumping, utils, dataset, decomposition

sysname = os.path.split(os.getcwd())[-1]
dt = 0.25
tica_lagtime = 400
regularization_string = "_012"

X0 = dataset.dataset("./tica/tica%d%s.h5" % (tica_lagtime, regularization_string))

slicer = featurizer.FirstSlicer(2)
X = slicer.transform(X0)

Xf0 = np.concatenate(X)
Xf = Xf0[::50]

hexbin(Xf0[:, 0], Xf0[:, 1], bins='log')

svm = OneClassSVM(nu=0.15)
svm.fit(Xf)

y = svm.predict(Xf)

plot(Xf[y==1][:, 0], Xf[y==1][:, 1], 'kx')
plot(Xf[y==-1][:, 0], Xf[y==-1][:, 1], 'wx')

clusterer = cluster.GMM(n_components=3)

yi = map(lambda x: svm.predict(x), X)

from msmbuilder.cluster import MultiSequenceClusterMixin, BaseEstimator
from sklearn.svm import OneClassSVM

class OneClassSVMTrimmer(MultiSequenceClusterMixin, OneClassSVM, BaseEstimator):
    def partial_transform(self, traj):
        """Featurize an MD trajectory into a vector space.

        Parameters
        ----------
        traj : mdtraj.Trajectory
            A molecular dynamics trajectory to featurize.

        Returns
        -------
        features : np.ndarray, dtype=float, shape=(n_samples, n_features)
            A featurized trajectory is a 2D array of shape
            `(length_of_trajectory x n_features)` where each `features[i]`
            vector is computed by applying the featurization function
            to the `i`th snapshot of the input trajectory.

        See Also
        --------
        transform : simultaneously featurize a collection of MD trajectories
        """
        pass

    def transform(self, traj_list, y=None):
        """Featurize a several trajectories.

        Parameters
        ----------
        traj_list : list(mdtraj.Trajectory)
            Trajectories to be featurized.

        Returns
        -------
        features : list(np.ndarray), length = len(traj_list)
            The featurized trajectories.  features[i] is the featurized
            version of traj_list[i] and has shape
            (n_samples_i, n_features)
        """
        return [self.partial_transform(traj) for traj in traj_list]

trimmer = OneClassSVMTrimmer()
trimmer.fit(X[0:10])


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Short interest dynamics

"""
from __future__ import print_function, division

import os
import zipfile

import datetime as dt
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

path = os.getenv("HOME") + '/Dropbox/Research/data/Compustat/data/'
# __location__ = os.path.realpath(os.path.join(os.getcwd(),
#                                 os.path.dirname(__file__)))
# path = os.path.join(__location__, path + 'Compustat/data/')

def date_convert(string):
    return dt.datetime.strptime(string, '%d-%m-%Y')

def import_data():
    """Import data and save it to the disk.

    """
    zf = zipfile.ZipFile(path + 'short_int.zip', 'r')
    name = zf.namelist()[0]

    short_int = pd.read_csv(zf.open(name),
                            converters={'datadate': date_convert})
    columns = {'datadate': 'date',
               'SHORTINTADJ': 'short_int',
               'GVKEY': 'gvkey'}
    short_int.rename(columns=columns, inplace=True)
    short_int.set_index(['gvkey', 'date'], inplace=True)
    short_int.sort_index(inplace=True)

    short_int.to_hdf(path + 'short_int.h5', key='short_int')

    print(short_int.head())
    print(short_int.dtypes)
    print('Number of unique companies: ',
          short_int.index.get_level_values('gvkey').nunique())
    print('Number of unique dates: ',
          short_int.index.get_level_values('date').nunique())
    print('Min and Max date: ',
          short_int.index.get_level_values('date').min().date(), ',',
          short_int.index.get_level_values('date').max().date())

def load_data():
    """Load data from disk and check for sanity.

    """
    return pd.read_hdf(path + 'short_int.h5', 'short_int')

def count_companies(short_int):
    """Plot number of companies over time.

    """
    df = short_int.reset_index().groupby('date')['gvkey'].nunique()

    sns.set_context('paper')

    df.plot(figsize=(10, 3))
    plt.show()

    data = df.ix[dt.date(2006, 1, 1):dt.date(2007, 6, 30)]
    data.plot(figsize=(10, 3))
    plt.show()

def mean_short_int(short_int):
    """Mean short interest on each date.

    """
    df = short_int.groupby(level='date')['short_int'].mean()

    sns.set_context('paper')

    df.plot(figsize=(10, 3))
    plt.show()

    df.ix[:dt.date(2004, 12, 31)].plot(figsize=(10, 3))
    plt.show()

    df.ix[dt.date(2006, 1, 1):dt.date(2007, 6, 30)].plot(figsize=(10, 3))
    plt.show()

if __name__ == '__main__':

    import_data()
    short_int = load_data()
    count_companies(short_int)
    mean_short_int(short_int)


import itertools
import pickle

import numpy as np
from numpy.testing import assert_array_almost_equal

import scipy
from scipy.spatial.distance import cdist
from sklearn.neighbors.dist_metrics import DistanceMetric
from nose import SkipTest

def dist_func(x1, x2, p):
    return np.sum((x1 - x2) ** p) ** (1. / p)

def cmp_version(version1, version2):
    version1 = tuple(map(int, version1.split('.')[:2]))
    version2 = tuple(map(int, version2.split('.')[:2]))

    if version1 < version2:
        return -1
    elif version1 > version2:
        return 1
    else:
        return 0

class TestMetrics:
    def __init__(self, n1=20, n2=25, d=4, zero_frac=0.5,
                 rseed=0, dtype=np.float64):
        np.random.seed(rseed)
        self.X1 = np.random.random((n1, d)).astype(dtype)
        self.X2 = np.random.random((n2, d)).astype(dtype)

        # make boolean arrays: ones and zeros
        self.X1_bool = self.X1.round(0)
        self.X2_bool = self.X2.round(0)

        V = np.random.random((d, d))
        VI = np.dot(V, V.T)

        self.metrics = {'euclidean': {},
                        'cityblock': {},
                        'minkowski': dict(p=(1, 1.5, 2, 3)),
                        'chebyshev': {},
                        'seuclidean': dict(V=(np.random.random(d),)),
                        'wminkowski': dict(p=(1, 1.5, 3),
                                           w=(np.random.random(d),)),
                        'mahalanobis': dict(VI=(VI,)),
                        'hamming': {},
                        'canberra': {},
                        'braycurtis': {}}

        self.bool_metrics = ['matching', 'jaccard', 'dice',
                             'kulsinski', 'rogerstanimoto', 'russellrao',
                             'sokalmichener', 'sokalsneath']

    def test_cdist(self):
        for metric, argdict in self.metrics.items():
            keys = argdict.keys()
            for vals in itertools.product(*argdict.values()):
                kwargs = dict(zip(keys, vals))
                D_true = cdist(self.X1, self.X2, metric, **kwargs)
                yield self.check_cdist, metric, kwargs, D_true

        for metric in self.bool_metrics:
            D_true = cdist(self.X1_bool, self.X2_bool, metric)
            yield self.check_cdist_bool, metric, D_true

    def check_cdist(self, metric, kwargs, D_true):
        if metric == 'canberra' and cmp_version(scipy.__version__, '0.9') <= 0:
            raise SkipTest("Canberra distance incorrect in scipy < 0.9")
        dm = DistanceMetric.get_metric(metric, **kwargs)
        D12 = dm.pairwise(self.X1, self.X2)
        assert_array_almost_equal(D12, D_true)

    def check_cdist_bool(self, metric, D_true):
        dm = DistanceMetric.get_metric(metric)
        D12 = dm.pairwise(self.X1_bool, self.X2_bool)
        assert_array_almost_equal(D12, D_true)

    def test_pdist(self):
        for metric, argdict in self.metrics.items():
            keys = argdict.keys()
            for vals in itertools.product(*argdict.values()):
                kwargs = dict(zip(keys, vals))
                D_true = cdist(self.X1, self.X1, metric, **kwargs)
                yield self.check_pdist, metric, kwargs, D_true

        for metric in self.bool_metrics:
            D_true = cdist(self.X1_bool, self.X1_bool, metric)
            yield self.check_pdist_bool, metric, D_true

    def check_pdist(self, metric, kwargs, D_true):
        if metric == 'canberra' and cmp_version(scipy.__version__, '0.9') <= 0:
            raise SkipTest("Canberra distance incorrect in scipy < 0.9")
        dm = DistanceMetric.get_metric(metric, **kwargs)
        D12 = dm.pairwise(self.X1)
        assert_array_almost_equal(D12, D_true)

    def check_pdist_bool(self, metric, D_true):
        dm = DistanceMetric.get_metric(metric)
        D12 = dm.pairwise(self.X1_bool)
        assert_array_almost_equal(D12, D_true)

def test_haversine_metric():
    def haversine_slow(x1, x2):
        return 2 * np.arcsin(np.sqrt(np.sin(0.5 * (x1[0] - x2[0])) ** 2
                                     + np.cos(x1[0]) * np.cos(x2[0]) *
                                     np.sin(0.5 * (x1[1] - x2[1])) ** 2))

    X = np.random.random((10, 2))

    haversine = DistanceMetric.get_metric("haversine")

    D1 = haversine.pairwise(X)
    D2 = np.zeros_like(D1)
    for i, x1 in enumerate(X):
        for j, x2 in enumerate(X):
            D2[i, j] = haversine_slow(x1, x2)

    assert_array_almost_equal(D1, D2)
    assert_array_almost_equal(haversine.dist_to_rdist(D1),
                              np.sin(0.5 * D2) ** 2)

def test_pyfunc_metric():
    X = np.random.random((10, 3))

    euclidean = DistanceMetric.get_metric("euclidean")
    pyfunc = DistanceMetric.get_metric("pyfunc", func=dist_func, p=2)

    # Check if both callable metric and predefined metric initialized
    # DistanceMetric object is picklable
    euclidean_pkl = pickle.loads(pickle.dumps(euclidean))
    pyfunc_pkl = pickle.loads(pickle.dumps(pyfunc))

    D1 = euclidean.pairwise(X)
    D2 = pyfunc.pairwise(X)

    D1_pkl = euclidean_pkl.pairwise(X)
    D2_pkl = pyfunc_pkl.pairwise(X)

    assert_array_almost_equal(D1, D2)
    assert_array_almost_equal(D1_pkl, D2_pkl)