# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A dataset loader for imports85.data."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

import numpy as np
import tensorflow as tf

try:
  import pandas as pd  # pylint: disable=g-import-not-at-top
except ImportError:
  pass

URL = "https://archive.ics.uci.edu/ml/machine-learning-databases/autos/imports-85.data"

# Order is important for the csv-readers, so we use an OrderedDict here.
defaults = collections.OrderedDict([
    ("symboling", [0]),
    ("normalized-losses", [0.0]),
    ("make", [""]),
    ("fuel-type", [""]),
    ("aspiration", [""]),
    ("num-of-doors", [""]),
    ("body-style", [""]),
    ("drive-wheels", [""]),
    ("engine-location", [""]),
    ("wheel-base", [0.0]),
    ("length", [0.0]),
    ("width", [0.0]),
    ("height", [0.0]),
    ("curb-weight", [0.0]),
    ("engine-type", [""]),
    ("num-of-cylinders", [""]),
    ("engine-size", [0.0]),
    ("fuel-system", [""]),
    ("bore", [0.0]),
    ("stroke", [0.0]),
    ("compression-ratio", [0.0]),
    ("horsepower", [0.0]),
    ("peak-rpm", [0.0]),
    ("city-mpg", [0.0]),
    ("highway-mpg", [0.0]),
    ("price", [0.0])
])  # pyformat: disable

types = collections.OrderedDict((key, type(value[0]))
                                for key, value in defaults.items())

def _get_imports85():
  path = tf.contrib.keras.utils.get_file(URL.split("/")[-1], URL)
  return path

def dataset(y_name="price", train_fraction=0.7):
  """Load the imports85 data as a (train,test) pair of `Dataset`.

  Each dataset generates (features_dict, label) pairs.

  Args:
    y_name: The name of the column to use as the label.
    train_fraction: A float, the fraction of data to use for training. The
        remainder will be used for evaluation.
  Returns:
    A (train,test) pair of `Datasets`
  """
  # Download and cache the data
  path = _get_imports85()

  # Define how the lines of the file should be parsed
  def decode_line(line):
    """Convert a csv line into a (features_dict,label) pair."""
    # Decode the line to a tuple of items based on the types of
    # csv_header.values().
    items = tf.decode_csv(line, list(defaults.values()))

    # Convert the keys and items to a dict.
    pairs = zip(defaults.keys(), items)
    features_dict = dict(pairs)

    # Remove the label from the features_dict
    label = features_dict.pop(y_name)

    return features_dict, label

  def has_no_question_marks(line):
    """Returns True if the line of text has no question marks."""
    # split the line into an array of characters
    chars = tf.string_split(line[tf.newaxis], "").values
    # for each character check if it is a question mark
    is_question = tf.equal(chars, "?")
    any_question = tf.reduce_any(is_question)
    no_question = ~any_question

    return no_question

  def in_training_set(line):
    """Returns a boolean tensor, true if the line is in the training set."""
    # If you randomly split the dataset you won't get the same split in both
    # sessions if you stop and restart training later. Also a simple
    # random split won't work with a dataset that's too big to `.cache()` as
    # we are doing here.
    num_buckets = 1000000
    bucket_id = tf.string_to_hash_bucket_fast(line, num_buckets)
    # Use the hash bucket id as a random number that's deterministic per example
    return bucket_id < int(train_fraction * num_buckets)

  def in_test_set(line):
    """Returns a boolean tensor, true if the line is in the training set."""
    # Items not in the training set are in the test set.
    # This line must use `~` instead of `not` because `not` only works on python
    # booleans but we are dealing with symbolic tensors.
    return ~in_training_set(line)

  base_dataset = (tf.contrib.data
                  # Get the lines from the file.
                  .TextLineDataset(path)
                  # drop lines with question marks.
                  .filter(has_no_question_marks))

  train = (base_dataset
           # Take only the training-set lines.
           .filter(in_training_set)
           # Decode each line into a (features_dict, label) pair.
           .map(decode_line)
           # Cache data so you only decode the file once.
           .cache())

  # Do the same for the test-set.
  test = (base_dataset.filter(in_test_set).cache().map(decode_line))

  return train, test

def raw_dataframe():
  """Load the imports85 data as a pd.DataFrame."""
  # Download and cache the data
  path = _get_imports85()

  # Load it into a pandas dataframe
  df = pd.read_csv(path, names=types.keys(), dtype=types, na_values="?")

  return df

def load_data(y_name="price", train_fraction=0.7, seed=None):
  """Get the imports85 data set.

  A description of the data is available at:
    https://archive.ics.uci.edu/ml/datasets/automobile

  The data itself can be found at:
    https://archive.ics.uci.edu/ml/machine-learning-databases/autos/imports-85.data

  Args:
    y_name: the column to return as the label.
    train_fraction: the fraction of the dataset to use for training.
    seed: The random seed to use when shuffling the data. `None` generates a
      unique shuffle every run.
  Returns:
    a pair of pairs where the first pair is the training data, and the second
    is the test data:
    `(x_train, y_train), (x_test, y_test) = get_imports85_dataset(...)`
    `x` contains a pandas DataFrame of features, while `y` contains the label
    array.
  """
  # Load the raw data columns.
  data = raw_dataframe()

  # Delete rows with unknowns
  data = data.dropna()

  # Shuffle the data
  np.random.seed(seed)

  # Split the data into train/test subsets.
  x_train = data.sample(frac=train_fraction, random_state=seed)
  x_test = data.drop(x_train.index)

  # Extract the label from the features dataframe.
  y_train = x_train.pop(y_name)
  y_test = x_test.pop(y_name)

  return (x_train, y_train), (x_test, y_test)


# -*- coding: utf-8 -*-

import re

import numpy as np
import pandas as pd
import pandas.util.testing as tm
import pytest
from pandas import IntervalIndex, MultiIndex, RangeIndex
from pandas.compat import lrange, range
from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike

def test_labels_dtypes():

    # GH 8456
    i = MultiIndex.from_tuples([('A', 1), ('A', 2)])
    assert i.labels[0].dtype == 'int8'
    assert i.labels[1].dtype == 'int8'

    i = MultiIndex.from_product([['a'], range(40)])
    assert i.labels[1].dtype == 'int8'
    i = MultiIndex.from_product([['a'], range(400)])
    assert i.labels[1].dtype == 'int16'
    i = MultiIndex.from_product([['a'], range(40000)])
    assert i.labels[1].dtype == 'int32'

    i = pd.MultiIndex.from_product([['a'], range(1000)])
    assert (i.labels[0] >= 0).all()
    assert (i.labels[1] >= 0).all()

def test_values_boxed():
    tuples = [(1, pd.Timestamp('2000-01-01')), (2, pd.NaT),
              (3, pd.Timestamp('2000-01-03')),
              (1, pd.Timestamp('2000-01-04')),
              (2, pd.Timestamp('2000-01-02')),
              (3, pd.Timestamp('2000-01-03'))]
    result = pd.MultiIndex.from_tuples(tuples)
    expected = construct_1d_object_array_from_listlike(tuples)
    tm.assert_numpy_array_equal(result.values, expected)
    # Check that code branches for boxed values produce identical results
    tm.assert_numpy_array_equal(result.values[:4], result[:4].values)

def test_values_multiindex_datetimeindex():
    # Test to ensure we hit the boxing / nobox part of MI.values
    ints = np.arange(10 ** 18, 10 ** 18 + 5)
    naive = pd.DatetimeIndex(ints)
    aware = pd.DatetimeIndex(ints, tz='US/Central')

    idx = pd.MultiIndex.from_arrays([naive, aware])
    result = idx.values

    outer = pd.DatetimeIndex([x[0] for x in result])
    tm.assert_index_equal(outer, naive)

    inner = pd.DatetimeIndex([x[1] for x in result])
    tm.assert_index_equal(inner, aware)

    # n_lev > n_lab
    result = idx[:2].values

    outer = pd.DatetimeIndex([x[0] for x in result])
    tm.assert_index_equal(outer, naive[:2])

    inner = pd.DatetimeIndex([x[1] for x in result])
    tm.assert_index_equal(inner, aware[:2])

def test_values_multiindex_periodindex():
    # Test to ensure we hit the boxing / nobox part of MI.values
    ints = np.arange(2007, 2012)
    pidx = pd.PeriodIndex(ints, freq='D')

    idx = pd.MultiIndex.from_arrays([ints, pidx])
    result = idx.values

    outer = pd.Int64Index([x[0] for x in result])
    tm.assert_index_equal(outer, pd.Int64Index(ints))

    inner = pd.PeriodIndex([x[1] for x in result])
    tm.assert_index_equal(inner, pidx)

    # n_lev > n_lab
    result = idx[:2].values

    outer = pd.Int64Index([x[0] for x in result])
    tm.assert_index_equal(outer, pd.Int64Index(ints[:2]))

    inner = pd.PeriodIndex([x[1] for x in result])
    tm.assert_index_equal(inner, pidx[:2])

def test_consistency():
    # need to construct an overflow
    major_axis = lrange(70000)
    minor_axis = lrange(10)

    major_labels = np.arange(70000)
    minor_labels = np.repeat(lrange(10), 7000)

    # the fact that is works means it's consistent
    index = MultiIndex(levels=[major_axis, minor_axis],
                       labels=[major_labels, minor_labels])

    # inconsistent
    major_labels = np.array([0, 0, 1, 1, 1, 2, 2, 3, 3])
    minor_labels = np.array([0, 1, 0, 1, 1, 0, 1, 0, 1])
    index = MultiIndex(levels=[major_axis, minor_axis],
                       labels=[major_labels, minor_labels])

    assert not index.is_unique

def test_hash_collisions():
    # non-smoke test that we don't get hash collisions

    index = MultiIndex.from_product([np.arange(1000), np.arange(1000)],
                                    names=['one', 'two'])
    result = index.get_indexer(index.values)
    tm.assert_numpy_array_equal(result, np.arange(
        len(index), dtype='intp'))

    for i in [0, 1, len(index) - 2, len(index) - 1]:
        result = index.get_loc(index[i])
        assert result == i

def test_dims():
    pass

def take_invalid_kwargs():
    vals = [['A', 'B'],
            [pd.Timestamp('2011-01-01'), pd.Timestamp('2011-01-02')]]
    idx = pd.MultiIndex.from_product(vals, names=['str', 'dt'])
    indices = [1, 2]

    msg = r"take\(\) got an unexpected keyword argument 'foo'"
    tm.assert_raises_regex(TypeError, msg, idx.take,
                           indices, foo=2)

    msg = "the 'out' parameter is not supported"
    tm.assert_raises_regex(ValueError, msg, idx.take,
                           indices, out=indices)

    msg = "the 'mode' parameter is not supported"
    tm.assert_raises_regex(ValueError, msg, idx.take,
                           indices, mode='clip')

def test_isna_behavior(idx):
    # should not segfault GH5123
    # NOTE: if MI representation changes, may make sense to allow
    # isna(MI)
    with pytest.raises(NotImplementedError):
        pd.isna(idx)

def test_large_multiindex_error():
    # GH12527
    df_below_1000000 = pd.DataFrame(
        1, index=pd.MultiIndex.from_product([[1, 2], range(499999)]),
        columns=['dest'])
    with pytest.raises(KeyError):
        df_below_1000000.loc[(-1, 0), 'dest']
    with pytest.raises(KeyError):
        df_below_1000000.loc[(3, 0), 'dest']
    df_above_1000000 = pd.DataFrame(
        1, index=pd.MultiIndex.from_product([[1, 2], range(500001)]),
        columns=['dest'])
    with pytest.raises(KeyError):
        df_above_1000000.loc[(-1, 0), 'dest']
    with pytest.raises(KeyError):
        df_above_1000000.loc[(3, 0), 'dest']

def test_million_record_attribute_error():
    # GH 18165
    r = list(range(1000000))
    df = pd.DataFrame({'a': r, 'b': r},
                      index=pd.MultiIndex.from_tuples([(x, x) for x in r]))

    with tm.assert_raises_regex(AttributeError,
                                "'Series' object has no attribute 'foo'"):
        df['a'].foo()

def test_can_hold_identifiers(idx):
    key = idx[0]
    assert idx._can_hold_identifiers_and_holds_name(key) is True

def test_metadata_immutable(idx):
    levels, labels = idx.levels, idx.labels
    # shouldn't be able to set at either the top level or base level
    mutable_regex = re.compile('does not support mutable operations')
    with tm.assert_raises_regex(TypeError, mutable_regex):
        levels[0] = levels[0]
    with tm.assert_raises_regex(TypeError, mutable_regex):
        levels[0][0] = levels[0][0]
    # ditto for labels
    with tm.assert_raises_regex(TypeError, mutable_regex):
        labels[0] = labels[0]
    with tm.assert_raises_regex(TypeError, mutable_regex):
        labels[0][0] = labels[0][0]
    # and for names
    names = idx.names
    with tm.assert_raises_regex(TypeError, mutable_regex):
        names[0] = names[0]

def test_level_setting_resets_attributes():
    ind = pd.MultiIndex.from_arrays([
        ['A', 'A', 'B', 'B', 'B'], [1, 2, 1, 2, 3]
    ])
    assert ind.is_monotonic
    ind.set_levels([['A', 'B'], [1, 3, 2]], inplace=True)
    # if this fails, probably didn't reset the cache correctly.
    assert not ind.is_monotonic

def test_rangeindex_fallback_coercion_bug():
    # GH 12893
    foo = pd.DataFrame(np.arange(100).reshape((10, 10)))
    bar = pd.DataFrame(np.arange(100).reshape((10, 10)))
    df = pd.concat({'foo': foo.stack(), 'bar': bar.stack()}, axis=1)
    df.index.names = ['fizz', 'buzz']

    str(df)
    expected = pd.DataFrame({'bar': np.arange(100),
                             'foo': np.arange(100)},
                            index=pd.MultiIndex.from_product(
                                [range(10), range(10)],
                                names=['fizz', 'buzz']))
    tm.assert_frame_equal(df, expected, check_like=True)

    result = df.index.get_level_values('fizz')
    expected = pd.Int64Index(np.arange(10), name='fizz').repeat(10)
    tm.assert_index_equal(result, expected)

    result = df.index.get_level_values('buzz')
    expected = pd.Int64Index(np.tile(np.arange(10), 10), name='buzz')
    tm.assert_index_equal(result, expected)

def test_hash_error(indices):
    index = indices
    tm.assert_raises_regex(TypeError, "unhashable type: %r" %
                           type(index).__name__, hash, indices)

def test_mutability(indices):
    if not len(indices):
        return
    pytest.raises(TypeError, indices.__setitem__, 0, indices[0])

def test_wrong_number_names(indices):
    def testit(ind):
        ind.names = ["apple", "banana", "carrot"]
    tm.assert_raises_regex(ValueError, "^Length", testit, indices)

def test_memory_usage(idx):
    result = idx.memory_usage()
    if len(idx):
        idx.get_loc(idx[0])
        result2 = idx.memory_usage()
        result3 = idx.memory_usage(deep=True)

        # RangeIndex, IntervalIndex
        # don't have engines
        if not isinstance(idx, (RangeIndex, IntervalIndex)):
            assert result2 > result

        if idx.inferred_type == 'object':
            assert result3 > result2

    else:

        # we report 0 for no-length
        assert result == 0

def test_nlevels(idx):
    assert idx.nlevels == 2


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# DCASE 2016::Sound Event Detection in Real-life Audio / Baseline System

import argparse
import csv
import math
import numpy
import textwrap
import warnings

from sklearn import mixture

from src.dataset import *
from src.evaluation import *
from src.features import *
from src.sound_event_detection import *

__version_info__ = ('1', '0', '1')
__version__ = '.'.join(__version_info__)

def main(argv):
    numpy.random.seed(123456)  # let's make randomization predictable

    parser = argparse.ArgumentParser(
        prefix_chars='-+',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent('''\
            DCASE 2016
            Task 3: Sound Event Detection in Real-life Audio
            Baseline System
            ---------------------------------------------
                Tampere University of Technology / Audio Research Group
                Author:  Toni Heittola ( toni.heittola@tut.fi )

            System description
                This is an baseline implementation for the D-CASE 2016, task 3 - Sound event detection in real life audio.
                The system has binary classifier for each included sound event class. The GMM classifier is trained with
                the positive and negative examples from the mixture signals, and classification is done between these
                two models as likelihood ratio. Acoustic features are MFCC+Delta+Acceleration (MFCC0 omitted).

        '''))

    parser.add_argument("-development", help="Use the system in the development mode", action='store_true',
                        default=False, dest='development')
    parser.add_argument("-challenge", help="Use the system in the challenge mode", action='store_true',
                        default=False, dest='challenge')

    parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)
    args = parser.parse_args()

    # Load parameters from config file
    parameter_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  os.path.splitext(os.path.basename(__file__))[0] + '.yaml')
    params = load_parameters(parameter_file)
    params = process_parameters(params)
    make_folders(params)

    title("DCASE 2016::Sound Event Detection in Real-life Audio / Baseline System")

    # Check if mode is defined
    if not (args.development or args.challenge):
        args.development = True
        args.challenge = False

    dataset_evaluation_mode = 'folds'
    if args.development and not args.challenge:
        print "Running system in development mode"
        dataset_evaluation_mode = 'folds'
    elif not args.development and args.challenge:
        print "Running system in challenge mode"
        dataset_evaluation_mode = 'full'

    # Get dataset container class
    dataset = eval(params['general']['development_dataset'])(data_path=params['path']['data'])

    # Fetch data over internet and setup the data
    # ==================================================
    if params['flow']['initialize']:
        dataset.fetch()

    # Extract features for all audio files in the dataset
    # ==================================================
    if params['flow']['extract_features']:
        section_header('Feature extraction [Development data]')

        # Collect files from evaluation sets
        files = []
        for fold in dataset.folds(mode=dataset_evaluation_mode):
            for item_id, item in enumerate(dataset.train(fold)):
                if item['file'] not in files:
                    files.append(item['file'])
            for item_id, item in enumerate(dataset.test(fold)):
                if item['file'] not in files:
                    files.append(item['file'])

        # Go through files and make sure all features are extracted
        do_feature_extraction(files=files,
                              dataset=dataset,
                              feature_path=params['path']['features'],
                              params=params['features'],
                              overwrite=params['general']['overwrite'])

        foot()

    # Prepare feature normalizers
    # ==================================================
    if params['flow']['feature_normalizer']:
        section_header('Feature normalizer [Development data]')

        do_feature_normalization(dataset=dataset,
                                 feature_normalizer_path=params['path']['feature_normalizers'],
                                 feature_path=params['path']['features'],
                                 dataset_evaluation_mode=dataset_evaluation_mode,
                                 overwrite=params['general']['overwrite'])

        foot()

    # System training
    # ==================================================
    if params['flow']['train_system']:
        section_header('System training    [Development data]')

        do_system_training(dataset=dataset,
                           model_path=params['path']['models'],
                           feature_normalizer_path=params['path']['feature_normalizers'],
                           feature_path=params['path']['features'],
                           hop_length_seconds=params['features']['hop_length_seconds'],
                           classifier_params=params['classifier']['parameters'],
                           dataset_evaluation_mode=dataset_evaluation_mode,
                           classifier_method=params['classifier']['method'],
                           overwrite=params['general']['overwrite']
                           )

        foot()

    # System evaluation in development mode
    if args.development and not args.challenge:

        # System testing
        # ==================================================
        if params['flow']['test_system']:
            section_header('System testing     [Development data]')

            do_system_testing(dataset=dataset,
                              result_path=params['path']['results'],
                              feature_path=params['path']['features'],
                              model_path=params['path']['models'],
                              feature_params=params['features'],
                              detector_params=params['detector'],
                              dataset_evaluation_mode=dataset_evaluation_mode,
                              classifier_method=params['classifier']['method'],
                              overwrite=params['general']['overwrite']
                              )
            foot()

        # System evaluation
        # ==================================================
        if params['flow']['evaluate_system']:
            section_header('System evaluation  [Development data]')

            do_system_evaluation(dataset=dataset,
                                 dataset_evaluation_mode=dataset_evaluation_mode,
                                 result_path=params['path']['results'])

            foot()

    # System evaluation with challenge data
    elif not args.development and args.challenge:
        # Fetch data over internet and setup the data
        challenge_dataset = eval(params['general']['challenge_dataset'])()

        if params['flow']['initialize']:
            challenge_dataset.fetch()

        # System testing
        if params['flow']['test_system']:
            section_header('System testing     [Challenge data]')

            do_system_testing(dataset=challenge_dataset,
                              result_path=params['path']['challenge_results'],
                              feature_path=params['path']['features'],
                              model_path=params['path']['models'],
                              feature_params=params['features'],
                              detector_params=params['detector'],
                              dataset_evaluation_mode=dataset_evaluation_mode,
                              classifier_method=params['classifier']['method'],
                              overwrite=True
                              )
            foot()

            print " "
            print "Your results for the challenge data are stored at [" + params['path']['challenge_results'] + "]"
            print " "

def process_parameters(params):
    """Parameter post-processing.

    Parameters
    ----------
    params : dict
        parameters in dict

    Returns
    -------
    params : dict
        processed parameters

    """

    params['features']['mfcc']['win_length'] = int(params['features']['win_length_seconds'] * params['features']['fs'])
    params['features']['mfcc']['hop_length'] = int(params['features']['hop_length_seconds'] * params['features']['fs'])

    # Copy parameters for current classifier method
    params['classifier']['parameters'] = params['classifier_parameters'][params['classifier']['method']]

    # Hash
    params['features']['hash'] = get_parameter_hash(params['features'])
    params['classifier']['hash'] = get_parameter_hash(params['classifier'])
    params['detector']['hash'] = get_parameter_hash(params['detector'])

    # Paths
    params['path']['data'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), params['path']['data'])
    params['path']['base'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), params['path']['base'])

    # Features
    params['path']['features_'] = params['path']['features']
    params['path']['features'] = os.path.join(params['path']['base'],
                                              params['path']['features'],
                                              params['features']['hash'])

    # Feature normalizers
    params['path']['feature_normalizers_'] = params['path']['feature_normalizers']
    params['path']['feature_normalizers'] = os.path.join(params['path']['base'],
                                                         params['path']['feature_normalizers'],
                                                         params['features']['hash'])

    # Models
    # Save parameters into folders to help manual browsing of files.
    params['path']['models_'] = params['path']['models']
    params['path']['models'] = os.path.join(params['path']['base'],
                                            params['path']['models'],
                                            params['features']['hash'],
                                            params['classifier']['hash'])

    # Results
    params['path']['results_'] = params['path']['results']
    params['path']['results'] = os.path.join(params['path']['base'],
                                             params['path']['results'],
                                             params['features']['hash'],
                                             params['classifier']['hash'],
                                             params['detector']['hash'])
    return params

def make_folders(params, parameter_filename='parameters.yaml'):
    """Create all needed folders, and saves parameters in yaml-file for easier manual browsing of data.

    Parameters
    ----------
    params : dict
        parameters in dict

    parameter_filename : str
        filename to save parameters used to generate the folder name

    Returns
    -------
    nothing

    """

    # Check that target path exists, create if not
    check_path(params['path']['features'])
    check_path(params['path']['feature_normalizers'])
    check_path(params['path']['models'])
    check_path(params['path']['results'])

    # Save parameters into folders to help manual browsing of files.

    # Features
    feature_parameter_filename = os.path.join(params['path']['features'], parameter_filename)
    if not os.path.isfile(feature_parameter_filename):
        save_parameters(feature_parameter_filename, params['features'])

    # Feature normalizers
    feature_normalizer_parameter_filename = os.path.join(params['path']['feature_normalizers'], parameter_filename)
    if not os.path.isfile(feature_normalizer_parameter_filename):
        save_parameters(feature_normalizer_parameter_filename, params['features'])

    # Models
    model_features_parameter_filename = os.path.join(params['path']['base'],
                                                     params['path']['models_'],
                                                     params['features']['hash'],
                                                     parameter_filename)
    if not os.path.isfile(model_features_parameter_filename):
        save_parameters(model_features_parameter_filename, params['features'])

    model_models_parameter_filename = os.path.join(params['path']['base'],
                                                   params['path']['models_'],
                                                   params['features']['hash'],
                                                   params['classifier']['hash'],
                                                   parameter_filename)
    if not os.path.isfile(model_models_parameter_filename):
        save_parameters(model_models_parameter_filename, params['classifier'])

    # Results
    # Save parameters into folders to help manual browsing of files.
    result_features_parameter_filename = os.path.join(params['path']['base'],
                                                      params['path']['results_'],
                                                      params['features']['hash'],
                                                      parameter_filename)
    if not os.path.isfile(result_features_parameter_filename):
        save_parameters(result_features_parameter_filename, params['features'])

    result_models_parameter_filename = os.path.join(params['path']['base'],
                                                    params['path']['results_'],
                                                    params['features']['hash'],
                                                    params['classifier']['hash'],
                                                    parameter_filename)
    if not os.path.isfile(result_models_parameter_filename):
        save_parameters(result_models_parameter_filename, params['classifier'])

    result_detector_parameter_filename = os.path.join(params['path']['base'],
                                                      params['path']['results_'],
                                                      params['features']['hash'],
                                                      params['classifier']['hash'],
                                                      params['detector']['hash'],
                                                      parameter_filename)
    if not os.path.isfile(result_detector_parameter_filename):
        save_parameters(result_detector_parameter_filename, params['detector'])

def get_feature_filename(audio_file, path, extension='cpickle'):
    """Get feature filename

    Parameters
    ----------
    audio_file : str
        audio file name from which the features are extracted

    path :  str
        feature path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    feature_filename : str
        full feature filename

    """

    return os.path.join(path, 'sequence_' + os.path.splitext(audio_file)[0] + '.' + extension)

def get_feature_normalizer_filename(fold, scene_label, path, extension='cpickle'):
    """Get normalizer filename

    Parameters
    ----------
    fold : int >= 0
        evaluation fold number

    scene_label : str
        scene label

    path :  str
        normalizer path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    normalizer_filename : str
        full normalizer filename

    """

    return os.path.join(path, 'scale_fold' + str(fold) + '_' + str(scene_label) + '.' + extension)

def get_model_filename(fold, scene_label, path, extension='cpickle'):
    """Get model filename

    Parameters
    ----------
    fold : int >= 0
        evaluation fold number

    scene_label : str
        scene label

    path :  str
        model path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    model_filename : str
        full model filename

    """

    return os.path.join(path, 'model_fold' + str(fold) + '_' + str(scene_label) + '.' + extension)

def get_result_filename(fold, scene_label, path, extension='txt'):
    """Get result filename

    Parameters
    ----------
    fold : int >= 0
        evaluation fold number

    scene_label : str
        scene label

    path :  str
        result path

    extension : str
        file extension
        (Default value='cpickle')

    Returns
    -------
    result_filename : str
        full result filename

    """

    if fold == 0:
        return os.path.join(path, 'results_' + str(scene_label) + '.' + extension)
    else:
        return os.path.join(path, 'results_fold' + str(fold) + '_' + str(scene_label) + '.' + extension)

def do_feature_extraction(files, dataset, feature_path, params, overwrite=False):
    """Feature extraction

    Parameters
    ----------
    files : list
        file list

    dataset : class
        dataset class

    feature_path : str
        path where the features are saved

    params : dict
        parameter dict

    overwrite : bool
        overwrite existing feature files
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    IOError
        Audio file not found.

    """

    for file_id, audio_filename in enumerate(files):
        # Get feature filename
        current_feature_file = get_feature_filename(audio_file=os.path.split(audio_filename)[1], path=feature_path)

        progress(title_text='Extracting [sequences]',
                 percentage=(float(file_id) / len(files)),
                 note=os.path.split(audio_filename)[1])

        if not os.path.isfile(current_feature_file) or overwrite:
            # Load audio
            if os.path.isfile(dataset.relative_to_absolute_path(audio_filename)):
                y, fs = load_audio(filename=dataset.relative_to_absolute_path(audio_filename), mono=True,
                                   fs=params['fs'])
            else:
                raise IOError("Audio file not found [%s]" % audio_filename)

            # Extract features
            feature_data = feature_extraction(y=y,
                                              fs=fs,
                                              include_mfcc0=params['include_mfcc0'],
                                              include_delta=params['include_delta'],
                                              include_acceleration=params['include_acceleration'],
                                              mfcc_params=params['mfcc'],
                                              delta_params=params['mfcc_delta'],
                                              acceleration_params=params['mfcc_acceleration'])
            # Save
            save_data(current_feature_file, feature_data)

def do_feature_normalization(dataset, feature_normalizer_path, feature_path, dataset_evaluation_mode='folds',
                             overwrite=False):
    """Feature normalization

    Calculated normalization factors for each evaluation fold based on the training material available.

    Parameters
    ----------
    dataset : class
        dataset class

    feature_normalizer_path : str
        path where the feature normalizers are saved.

    feature_path : str
        path where the features are saved.

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    overwrite : bool
        overwrite existing normalizers
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    IOError
        Feature file not found.

    """

    for fold in dataset.folds(mode=dataset_evaluation_mode):
        for scene_id, scene_label in enumerate(dataset.scene_labels):
            current_normalizer_file = get_feature_normalizer_filename(fold=fold, scene_label=scene_label,
                                                                      path=feature_normalizer_path)

            if not os.path.isfile(current_normalizer_file) or overwrite:
                # Collect sequence files from scene class
                files = []
                for item_id, item in enumerate(dataset.train(fold, scene_label=scene_label)):
                    if item['file'] not in files:
                        files.append(item['file'])

                file_count = len(files)

                # Initialize statistics
                normalizer = FeatureNormalizer()

                for file_id, audio_filename in enumerate(files):
                    progress(title_text='Collecting data',
                             fold=fold,
                             percentage=(float(file_id) / file_count),
                             note=os.path.split(audio_filename)[1])

                    # Load features
                    feature_filename = get_feature_filename(audio_file=os.path.split(audio_filename)[1],
                                                            path=feature_path)
                    if os.path.isfile(feature_filename):
                        feature_data = load_data(feature_filename)['stat']
                    else:
                        raise IOError("Feature file not found [%s]" % audio_filename)

                    # Accumulate statistics
                    normalizer.accumulate(feature_data)

                # Calculate normalization factors
                normalizer.finalize()

                # Save
                save_data(current_normalizer_file, normalizer)

def do_system_training(dataset, model_path, feature_normalizer_path, feature_path, hop_length_seconds,
                       classifier_params,
                       dataset_evaluation_mode='folds', classifier_method='gmm', overwrite=False):
    """System training

    Train a model pair for each sound event class, one for activity and one for inactivity.

    model container format:

    {
        'normalizer': normalizer class
        'models' :
            {
                'mouse click' :
                    {
                        'positive': mixture.GMM class,
                        'negative': mixture.GMM class
                    }
                'keyboard typing' :
                    {
                        'positive': mixture.GMM class,
                        'negative': mixture.GMM class
                    }
                ...
            }
    }

    Parameters
    ----------
    dataset : class
        dataset class

    model_path : str
        path where the models are saved.

    feature_normalizer_path : str
        path where the feature normalizers are saved.

    feature_path : str
        path where the features are saved.

    hop_length_seconds : float > 0
        feature frame hop length in seconds

    classifier_params : dict
        parameter dict

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    classifier_method : str ['gmm']
        classifier method, currently only GMM supported
        (Default value='gmm')

    overwrite : bool
        overwrite existing models
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    ValueError
        classifier_method is unknown.

    IOError
        Feature normalizer not found.
        Feature file not found.

    """

    if classifier_method != 'gmm':
        raise ValueError("Unknown classifier method [" + classifier_method + "]")

    for fold in dataset.folds(mode=dataset_evaluation_mode):
        for scene_id, scene_label in enumerate(dataset.scene_labels):
            current_model_file = get_model_filename(fold=fold, scene_label=scene_label, path=model_path)
            if not os.path.isfile(current_model_file) or overwrite:

                # Load normalizer
                feature_normalizer_filename = get_feature_normalizer_filename(fold=fold, scene_label=scene_label,
                                                                              path=feature_normalizer_path)
                if os.path.isfile(feature_normalizer_filename):
                    normalizer = load_data(feature_normalizer_filename)
                else:
                    raise IOError("Feature normalizer not found [%s]" % feature_normalizer_filename)

                # Initialize model container
                model_container = {'normalizer': normalizer, 'models': {}}

                # Restructure training data in to structure[files][events]
                ann = {}
                for item_id, item in enumerate(dataset.train(fold=fold, scene_label=scene_label)):
                    filename = os.path.split(item['file'])[1]
                    if filename not in ann:
                        ann[filename] = {}
                    if item['event_label'] not in ann[filename]:
                        ann[filename][item['event_label']] = []
                    ann[filename][item['event_label']].append((item['event_onset'], item['event_offset']))

                # Collect training examples
                data_positive = {}
                data_negative = {}
                file_count = len(ann)
                for item_id, audio_filename in enumerate(ann):
                    progress(title_text='Collecting data',
                             fold=fold,
                             percentage=(float(item_id) / file_count),
                             note=scene_label + " / " + os.path.split(audio_filename)[1])

                    # Load features
                    feature_filename = get_feature_filename(audio_file=audio_filename, path=feature_path)
                    if os.path.isfile(feature_filename):
                        feature_data = load_data(feature_filename)['feat']
                    else:
                        raise IOError("Feature file not found [%s]" % feature_filename)

                    # Normalize features
                    feature_data = model_container['normalizer'].normalize(feature_data)

                    for event_label in ann[audio_filename]:
                        positive_mask = numpy.zeros((feature_data.shape[0]), dtype=bool)

                        for event in ann[audio_filename][event_label]:
                            start_frame = int(math.floor(event[0] / hop_length_seconds))
                            stop_frame = int(math.ceil(event[1] / hop_length_seconds))

                            if stop_frame > feature_data.shape[0]:
                                stop_frame = feature_data.shape[0]

                            positive_mask[start_frame:stop_frame] = True

                        # Store positive examples
                        if event_label not in data_positive:
                            data_positive[event_label] = feature_data[positive_mask, :]
                        else:
                            data_positive[event_label] = numpy.vstack(
                                (data_positive[event_label], feature_data[positive_mask, :]))

                        # Store negative examples
                        if event_label not in data_negative:
                            data_negative[event_label] = feature_data[~positive_mask, :]
                        else:
                            data_negative[event_label] = numpy.vstack(
                                (data_negative[event_label], feature_data[~positive_mask, :]))

                # Train models for each class
                for event_label in data_positive:
                    progress(title_text='Train models',
                             fold=fold,
                             note=scene_label + " / " + event_label)
                    if classifier_method == 'gmm':
                        model_container['models'][event_label] = {}
                        model_container['models'][event_label]['positive'] = mixture.GMM(**classifier_params).fit(
                            data_positive[event_label])
                        model_container['models'][event_label]['negative'] = mixture.GMM(**classifier_params).fit(
                            data_negative[event_label])
                    else:
                        raise ValueError("Unknown classifier method [" + classifier_method + "]")

                # Save models
                save_data(current_model_file, model_container)

def do_system_testing(dataset, result_path, feature_path, model_path, feature_params, detector_params,
                      dataset_evaluation_mode='folds', classifier_method='gmm', overwrite=False):
    """System testing.

    If extracted features are not found from disk, they are extracted but not saved.

    Parameters
    ----------
    dataset : class
        dataset class

    result_path : str
        path where the results are saved.

    feature_path : str
        path where the features are saved.

    model_path : str
        path where the models are saved.

    feature_params : dict
        parameter dict

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    classifier_method : str ['gmm']
        classifier method, currently only GMM supported
        (Default value='gmm')

    overwrite : bool
        overwrite existing models
        (Default value=False)

    Returns
    -------
    nothing

    Raises
    -------
    ValueError
        classifier_method is unknown.

    IOError
        Model file not found.
        Audio file not found.

    """

    if classifier_method != 'gmm':
        raise ValueError("Unknown classifier method [" + classifier_method + "]")

    for fold in dataset.folds(mode=dataset_evaluation_mode):
        for scene_id, scene_label in enumerate(dataset.scene_labels):
            current_result_file = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)
            if not os.path.isfile(current_result_file) or overwrite:
                results = []

                # Load class model container
                model_filename = get_model_filename(fold=fold, scene_label=scene_label, path=model_path)
                if os.path.isfile(model_filename):
                    model_container = load_data(model_filename)
                else:
                    raise IOError("Model file not found [%s]" % model_filename)

                file_count = len(dataset.test(fold, scene_label=scene_label))
                for file_id, item in enumerate(dataset.test(fold=fold, scene_label=scene_label)):
                    progress(title_text='Testing',
                             fold=fold,
                             percentage=(float(file_id) / file_count),
                             note=scene_label + " / " + os.path.split(item['file'])[1])

                    # Load features
                    feature_filename = get_feature_filename(audio_file=item['file'], path=feature_path)

                    if os.path.isfile(feature_filename):
                        feature_data = load_data(feature_filename)['feat']
                    else:
                        # Load audio
                        if os.path.isfile(dataset.relative_to_absolute_path(item['file'])):
                            y, fs = load_audio(filename=item['file'], mono=True, fs=feature_params['fs'])
                        else:
                            raise IOError("Audio file not found [%s]" % item['file'])

                        # Extract features
                        feature_data = feature_extraction(y=y,
                                                          fs=fs,
                                                          include_mfcc0=feature_params['include_mfcc0'],
                                                          include_delta=feature_params['include_delta'],
                                                          include_acceleration=feature_params['include_acceleration'],
                                                          mfcc_params=feature_params['mfcc'],
                                                          delta_params=feature_params['mfcc_delta'],
                                                          acceleration_params=feature_params['mfcc_acceleration'],
                                                          statistics=False)['feat']

                    # Normalize features
                    feature_data = model_container['normalizer'].normalize(feature_data)

                    current_results = event_detection(feature_data=feature_data,
                                                      model_container=model_container,
                                                      hop_length_seconds=feature_params['hop_length_seconds'],
                                                      smoothing_window_length_seconds=detector_params[
                                                          'smoothing_window_length'],
                                                      decision_threshold=detector_params['decision_threshold'],
                                                      minimum_event_length=detector_params['minimum_event_length'],
                                                      minimum_event_gap=detector_params['minimum_event_gap'])

                    # Store the result
                    for event in current_results:
                        results.append((dataset.absolute_to_relative(item['file']), event[0], event[1], event[2]))

                # Save testing results
                with open(current_result_file, 'wt') as f:
                    writer = csv.writer(f, delimiter='\t')
                    for result_item in results:
                        writer.writerow(result_item)

def do_system_evaluation(dataset, result_path, dataset_evaluation_mode='folds'):
    """System evaluation. Testing outputs are collected and evaluated. Evaluation results are printed.

    Parameters
    ----------
    dataset : class
        dataset class

    result_path : str
        path where the results are saved.

    dataset_evaluation_mode : str ['folds', 'full']
        evaluation mode, 'full' all material available is considered to belong to one fold.
        (Default value='folds')

    Returns
    -------
    nothing

    Raises
    -------
    IOError
        Result file not found

    """

    # Set warnings off, sklearn metrics will trigger warning for classes without
    # predicted samples in F1-scoring. This is just to keep printing clean.
    warnings.simplefilter("ignore")

    overall_metrics_per_scene = {}

    for scene_id, scene_label in enumerate(dataset.scene_labels):
        if scene_label not in overall_metrics_per_scene:
            overall_metrics_per_scene[scene_label] = {}

        dcase2016_segment_based_metric = DCASE2016_EventDetection_SegmentBasedMetrics(
            class_list=dataset.event_labels(scene_label=scene_label))
        dcase2016_event_based_metric = DCASE2016_EventDetection_EventBasedMetrics(
            class_list=dataset.event_labels(scene_label=scene_label))

        for fold in dataset.folds(mode=dataset_evaluation_mode):
            results = []
            result_filename = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)

            if os.path.isfile(result_filename):
                with open(result_filename, 'rt') as f:
                    for row in csv.reader(f, delimiter='\t'):
                        results.append(row)
            else:
                raise IOError("Result file not found [%s]" % result_filename)

            for file_id, item in enumerate(dataset.test(fold, scene_label=scene_label)):
                current_file_results = []
                for result_line in results:
                    if len(result_line) != 0 and result_line[0] == dataset.absolute_to_relative(item['file']):
                        current_file_results.append(
                            {'file': result_line[0],
                             'event_onset': float(result_line[1]),
                             'event_offset': float(result_line[2]),
                             'event_label': result_line[3].rstrip()
                             }
                        )
                meta = dataset.file_meta(dataset.absolute_to_relative(item['file']))

                dcase2016_segment_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)
                dcase2016_event_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)

        overall_metrics_per_scene[scene_label]['segment_based_metrics'] = dcase2016_segment_based_metric.results()
        overall_metrics_per_scene[scene_label]['event_based_metrics'] = dcase2016_event_based_metric.results()

    print "  Evaluation over %d folds" % dataset.fold_count
    print " "
    print "  Results per scene "
    print "  {:18s} | {:5s} |  | {:39s}  ".format('', 'Main', 'Secondary metrics')
    print "  {:18s} | {:5s} |  | {:38s} | {:14s} | {:14s} |  {:14s} ".format('', '', 'Seg/Overall', 'Seg/Class',
                                                                             'Event/Overall', 'Event/Class')
    print "  {:18s} | {:5s} |  | {:6s} : {:5s} : {:5s} : {:5s} : {:5s} | {:6s} : {:5s} | {:6s} : {:5s} | {:6s} : {:5s} |".format(
        'Scene', 'ER', 'F1', 'ER', 'ER/S', 'ER/D', 'ER/I', 'F1', 'ER', 'F1', 'ER', 'F1', 'ER')
    print "  -------------------+-------+  +--------+-------+-------+-------+-------+--------+-------+--------+-------+--------+-------+"
    averages = {
        'segment_based_metrics': {
            'overall': {
                'ER': [],
                'F': [],
            },
            'class_wise_average': {
                'ER': [],
                'F': [],
            }
        },
        'event_based_metrics': {
            'overall': {
                'ER': [],
                'F': [],
            },
            'class_wise_average': {
                'ER': [],
                'F': [],
            }
        },
    }
    for scene_id, scene_label in enumerate(dataset.scene_labels):
        print "  {:18s} | {:5.2f} |  | {:4.1f} % : {:5.2f} : {:5.2f} : {:5.2f} : {:5.2f} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} |".format(
            scene_label,
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['ER'],
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['F'] * 100,
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['ER'],
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['S'],
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['D'],
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['I'],
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['F'] * 100,
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['ER'],
            overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['F'] * 100,
            overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['ER'],
            overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['F'] * 100,
            overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['ER'],
            )
        averages['segment_based_metrics']['overall']['ER'].append(
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['ER'])
        averages['segment_based_metrics']['overall']['F'].append(
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['F'])
        averages['segment_based_metrics']['class_wise_average']['ER'].append(
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['ER'])
        averages['segment_based_metrics']['class_wise_average']['F'].append(
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['F'])
        averages['event_based_metrics']['overall']['ER'].append(
            overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['ER'])
        averages['event_based_metrics']['overall']['F'].append(
            overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['F'])
        averages['event_based_metrics']['class_wise_average']['ER'].append(
            overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['ER'])
        averages['event_based_metrics']['class_wise_average']['F'].append(
            overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['F'])

    print "  -------------------+-------+  +--------+-------+-------+-------+-------+--------+-------+--------+-------+--------+-------+"
    print "  {:18s} | {:5.2f} |  | {:4.1f} % : {:5.2f} : {:21s} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} |".format(
        'Average',
        numpy.mean(averages['segment_based_metrics']['overall']['ER']),
        numpy.mean(averages['segment_based_metrics']['overall']['F']) * 100,
        numpy.mean(averages['segment_based_metrics']['overall']['ER']),
        ' ',
        numpy.mean(averages['segment_based_metrics']['class_wise_average']['F']) * 100,
        numpy.mean(averages['segment_based_metrics']['class_wise_average']['ER']),
        numpy.mean(averages['event_based_metrics']['overall']['F']) * 100,
        numpy.mean(averages['event_based_metrics']['overall']['ER']),
        numpy.mean(averages['event_based_metrics']['class_wise_average']['F']) * 100,
        numpy.mean(averages['event_based_metrics']['class_wise_average']['ER']),
        )

    print "  "
    # Restore warnings to default settings
    warnings.simplefilter("default")
    print "  Results per events "

    for scene_id, scene_label in enumerate(dataset.scene_labels):
        print "  "
        print "  " + scene_label.upper()
        print "  {:20s} | {:30s} |  | {:15s} ".format('', 'Segment-based', 'Event-based')
        print "  {:20s} | {:5s} : {:5s} : {:6s} : {:5s} |  | {:5s} : {:5s} : {:6s} : {:5s} |".format('Event', 'Nref',
                                                                                'Nsys', 'F1', 'ER',
                                                                                'Nref', 'Nsys',
                                                                                'F1', 'ER')
        print "  ---------------------+-------+-------+--------+-------+  +-------+-------+--------+-------+"
        seg_Nref = 0
        seg_Nsys = 0

        event_Nref = 0
        event_Nsys = 0
        for event_label in sorted(overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise']):
            print "  {:20s} | {:5d} : {:5d} : {:4.1f} % : {:5.2f} |  | {:5d} : {:5d} : {:4.1f} % : {:5.2f} |".format(
                event_label,
                int(overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nref']),
                int(overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nsys']),
                overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['F'] * 100,
                overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['ER'],
                int(overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nref']),
                int(overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nsys']),
                overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['F'] * 100,
                overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['ER'])
            seg_Nref += int(
                overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nref'])
            seg_Nsys += int(
                overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nsys'])

            event_Nref += int(
                overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nref'])
            event_Nsys += int(
                overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nsys'])
        print "  ---------------------+-------+-------+--------+-------+  +-------+-------+--------+-------+"
        print "  {:20s} | {:5d} : {:5d} : {:14s} |  | {:5d} : {:5d} : {:14s} |".format('Sum',
                                                                                seg_Nref,
                                                                                seg_Nsys,
                                                                                '',
                                                                                event_Nref,
                                                                                event_Nsys,
                                                                                '')
        print "  {:20s} | {:5s}   {:5s} : {:4.1f} % : {:5.2f} |  | {:5s}   {:5s} : {:4.1f} % : {:5.2f} |".format(
            'Average',
            '', '',
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['F'] * 100,
            overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['ER'],
            '', '',
            overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['F'] * 100,
            overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['ER'])
        print "  "

if __name__ == "__main__":
    try:
        sys.exit(main(sys.argv))
    except (ValueError, IOError) as e:
        sys.exit(e)