#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import numpy
import math
from sklearn import metrics

class DCASE2016_SceneClassification_Metrics():
    """DCASE 2016 scene classification metrics

    Examples
    --------

        >>> dcase2016_scene_metric = DCASE2016_SceneClassification_Metrics(class_list=dataset.scene_labels)
        >>> for fold in dataset.folds(mode=dataset_evaluation_mode):
        >>>     results = []
        >>>     result_filename = get_result_filename(fold=fold, path=result_path)
        >>>
        >>>     if os.path.isfile(result_filename):
        >>>         with open(result_filename, 'rt') as f:
        >>>             for row in csv.reader(f, delimiter='\t'):
        >>>                 results.append(row)
        >>>
        >>>     y_true = []
        >>>     y_pred = []
        >>>     for result in results:
        >>>         y_true.append(dataset.file_meta(result[0])[0]['scene_label'])
        >>>         y_pred.append(result[1])
        >>>
        >>>     dcase2016_scene_metric.evaluate(system_output=y_pred, annotated_ground_truth=y_true)
        >>>
        >>> results = dcase2016_scene_metric.results()

    """

    def __init__(self, class_list):
        """__init__ method.

        Parameters
        ----------
        class_list : list
            Evaluated scene labels in the list

        """
        self.accuracies_per_class = None
        self.Nsys = None
        self.Nref = None
        self.class_list = class_list
        self.eps = numpy.spacing(1)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return self.results()

    def accuracies(self, y_true, y_pred, labels):
        """Calculate accuracy

        Parameters
        ----------
        y_true : numpy.array
            Ground truth array, list of scene labels

        y_pred : numpy.array
            System output array, list of scene labels

        labels : list
            list of scene labels

        Returns
        -------
        array : numpy.array [shape=(number of scene labels,)]
            Accuracy per scene label class

        """

        confusion_matrix = metrics.confusion_matrix(y_true=y_true, y_pred=y_pred, labels=labels).astype(float)
        #print confusion_matrix
        temp =  numpy.divide(numpy.diag(confusion_matrix), numpy.sum(confusion_matrix, 1)+self.eps)
        #print temp
        return temp

    def evaluate(self, annotated_ground_truth, system_output):
        """Evaluate system output and annotated ground truth pair.

        Use results method to get results.

        Parameters
        ----------
        annotated_ground_truth : numpy.array
            Ground truth array, list of scene labels

        system_output : numpy.array
            System output array, list of scene labels

        Returns
        -------
        nothing

        """

        accuracies_per_class = self.accuracies(y_pred=system_output, y_true=annotated_ground_truth, labels=self.class_list)

        if self.accuracies_per_class is None:
            self.accuracies_per_class = accuracies_per_class
        else:
            self.accuracies_per_class = numpy.vstack((self.accuracies_per_class, accuracies_per_class))

        Nref = numpy.zeros(len(self.class_list))
        Nsys = numpy.zeros(len(self.class_list))

        for class_id, class_label in enumerate(self.class_list):
            for item in system_output:
                if item == class_label:
                    Nsys[class_id] += 1

            for item in annotated_ground_truth:
                if item == class_label:
                    Nref[class_id] += 1

        if self.Nref is None:
            self.Nref = Nref
        else:
            self.Nref = numpy.vstack((self.Nref, Nref))

        if self.Nsys is None:
            self.Nsys = Nsys
        else:
            self.Nsys = numpy.vstack((self.Nsys, Nsys))

    def results(self):
        """Get results

        Outputs results in dict, format:

            {
                'class_wise_data':
                    {
                        'office': {
                            'Nsys': 10,
                            'Nref': 7,
                        },
                    }
                'class_wise_accuracy':
                    {
                        'office': 0.6,
                        'home': 0.4,
                    }
                'overall_accuracy': numpy.mean(self.accuracies_per_class)
                'Nsys': 100,
                'Nref': 100,
            }

        Parameters
        ----------
        nothing

        Returns
        -------
        results : dict
            Results dict

        """

        results = {
            'class_wise_data': {},
            'class_wise_accuracy': {},
            'overall_accuracy': numpy.mean(self.accuracies_per_class)
        }
        if len(self.Nsys.shape) == 2:
            results['Nsys'] = int(sum(sum(self.Nsys)))
            results['Nref'] = int(sum(sum(self.Nref)))
        else:
            results['Nsys'] = int(sum(self.Nsys))
            results['Nref'] = int(sum(self.Nref))

        for class_id, class_label in enumerate(self.class_list):
            if len(self.accuracies_per_class.shape) == 2:
                results['class_wise_accuracy'][class_label] = numpy.mean(self.accuracies_per_class[:, class_id])
                results['class_wise_data'][class_label] = {
                   'Nsys': int(sum(self.Nsys[:, class_id])),
                    'Nref': int(sum(self.Nref[:, class_id])),
                }
            else:
                results['class_wise_accuracy'][class_label] = numpy.mean(self.accuracies_per_class[class_id])
                results['class_wise_data'][class_label] = {
                   'Nsys': int(self.Nsys[class_id]),
                    'Nref': int(self.Nref[class_id]),
                }

        return results

class EventDetectionMetrics(object):
    """Baseclass for sound event metric classes.
    """

    def __init__(self, class_list):
        """__init__ method.

        Parameters
        ----------
        class_list : list
            List of class labels to be evaluated.

        """

        self.class_list = class_list
        self.eps = numpy.spacing(1)

    def max_event_offset(self, data):
        """Get maximum event offset from event list

        Parameters
        ----------
        data : list
            Event list, list of event dicts

        Returns
        -------
        max : float > 0
            Maximum event offset
        """

        max = 0
        for event in data:
            if event['event_offset'] > max:
                max = event['event_offset']
        return max

    def list_to_roll(self, data, time_resolution=0.01):
        """Convert event list into event roll.
        Event roll is binary matrix indicating event activity withing time segment defined by time_resolution.

        Parameters
        ----------
        data : list
            Event list, list of event dicts

        time_resolution : float > 0
            Time resolution used when converting event into event roll.

        Returns
        -------
        event_roll : numpy.ndarray [shape=(math.ceil(data_length * 1 / time_resolution) + 1, amount of classes)]
            Event roll
        """

        # Initialize
        data_length = self.max_event_offset(data)
        event_roll = numpy.zeros((math.ceil(data_length * 1 / time_resolution) + 1, len(self.class_list)))

        # Fill-in event_roll
        for event in data:
            pos = self.class_list.index(event['event_label'].rstrip())

            onset = math.floor(event['event_onset'] * 1 / time_resolution)
            offset = math.ceil(event['event_offset'] * 1 / time_resolution) + 1

            event_roll[onset:offset, pos] = 1

        return event_roll

class DCASE2016_EventDetection_SegmentBasedMetrics(EventDetectionMetrics):
    """DCASE2016 Segment based metrics for sound event detection

    Supported metrics:
    - Overall
        - Error rate (ER), Substitutions (S), Insertions (I), Deletions (D)
        - F-score (F1)
    - Class-wise
        - Error rate (ER), Insertions (I), Deletions (D)
        - F-score (F1)

    Examples
    --------

    >>> overall_metrics_per_scene = {}
    >>> for scene_id, scene_label in enumerate(dataset.scene_labels):
    >>>     dcase2016_segment_based_metric = DCASE2016_EventDetection_SegmentBasedMetrics(class_list=dataset.event_labels(scene_label=scene_label))
    >>>     for fold in dataset.folds(mode=dataset_evaluation_mode):
    >>>         results = []
    >>>         result_filename = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)
    >>>
    >>>         if os.path.isfile(result_filename):
    >>>             with open(result_filename, 'rt') as f:
    >>>                 for row in csv.reader(f, delimiter='\t'):
    >>>                     results.append(row)
    >>>
    >>>         for file_id, item in enumerate(dataset.test(fold,scene_label=scene_label)):
    >>>             current_file_results = []
    >>>             for result_line in results:
    >>>                 if result_line[0] == dataset.absolute_to_relative(item['file']):
    >>>                     current_file_results.append(
    >>>                         {'file': result_line[0],
    >>>                          'event_onset': float(result_line[1]),
    >>>                          'event_offset': float(result_line[2]),
    >>>                          'event_label': result_line[3]
    >>>                          }
    >>>                     )
    >>>             meta = dataset.file_meta(dataset.absolute_to_relative(item['file']))
    >>>         dcase2016_segment_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)
    >>> overall_metrics_per_scene[scene_label]['segment_based_metrics'] = dcase2016_segment_based_metric.results()

    """

    def __init__(self, class_list, time_resolution=1.0):
        """__init__ method.

        Parameters
        ----------
        class_list : list
            List of class labels to be evaluated.

        time_resolution : float > 0
            Time resolution used when converting event into event roll.
            (Default value = 1.0)

        """

        self.time_resolution = time_resolution

        self.overall = {
            'Ntp': 0.0,
            'Ntn': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
            'Nref': 0.0,
            'Nsys': 0.0,
            'ER': 0.0,
            'S': 0.0,
            'D': 0.0,
            'I': 0.0,
        }
        self.class_wise = {}

        for class_label in class_list:
            self.class_wise[class_label] = {
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
                'Nref': 0.0,
                'Nsys': 0.0,
            }

        EventDetectionMetrics.__init__(self, class_list=class_list)

    def __enter__(self):
        # Initialize class and return it
        return self

    def __exit__(self, type, value, traceback):
        # Finalize evaluation and return results
        return self.results()

    def evaluate(self, annotated_ground_truth, system_output):
        """Evaluate system output and annotated ground truth pair.

        Use results method to get results.

        Parameters
        ----------
        annotated_ground_truth : numpy.array
            Ground truth array, list of scene labels

        system_output : numpy.array
            System output array, list of scene labels

        Returns
        -------
        nothing

        """

        # Convert event list into frame-based representation
        system_event_roll = self.list_to_roll(data=system_output, time_resolution=self.time_resolution)
        annotated_event_roll = self.list_to_roll(data=annotated_ground_truth, time_resolution=self.time_resolution)

        # Fix durations of both event_rolls to be equal
        if annotated_event_roll.shape[0] > system_event_roll.shape[0]:
            padding = numpy.zeros((annotated_event_roll.shape[0] - system_event_roll.shape[0], len(self.class_list)))
            system_event_roll = numpy.vstack((system_event_roll, padding))

        if system_event_roll.shape[0] > annotated_event_roll.shape[0]:
            padding = numpy.zeros((system_event_roll.shape[0] - annotated_event_roll.shape[0], len(self.class_list)))
            annotated_event_roll = numpy.vstack((annotated_event_roll, padding))

        # Compute segment-based overall metrics
        for segment_id in range(0, annotated_event_roll.shape[0]):
            annotated_segment = annotated_event_roll[segment_id, :]
            system_segment = system_event_roll[segment_id, :]

            Ntp = sum(system_segment + annotated_segment > 1)
            Ntn = sum(system_segment + annotated_segment == 0)
            Nfp = sum(system_segment - annotated_segment > 0)
            Nfn = sum(annotated_segment - system_segment > 0)

            Nref = sum(annotated_segment)
            Nsys = sum(system_segment)

            S = min(Nref, Nsys) - Ntp
            D = max(0, Nref - Nsys)
            I = max(0, Nsys - Nref)
            ER = max(Nref, Nsys) - Ntp

            self.overall['Ntp'] += Ntp
            self.overall['Ntn'] += Ntn
            self.overall['Nfp'] += Nfp
            self.overall['Nfn'] += Nfn
            self.overall['Nref'] += Nref
            self.overall['Nsys'] += Nsys
            self.overall['S'] += S
            self.overall['D'] += D
            self.overall['I'] += I
            self.overall['ER'] += ER

        for class_id, class_label in enumerate(self.class_list):
            annotated_segment = annotated_event_roll[:, class_id]
            system_segment = system_event_roll[:, class_id]

            Ntp = sum(system_segment + annotated_segment > 1)
            Ntn = sum(system_segment + annotated_segment == 0)
            Nfp = sum(system_segment - annotated_segment > 0)
            Nfn = sum(annotated_segment - system_segment > 0)

            Nref = sum(annotated_segment)
            Nsys = sum(system_segment)

            self.class_wise[class_label]['Ntp'] += Ntp
            self.class_wise[class_label]['Ntn'] += Ntn
            self.class_wise[class_label]['Nfp'] += Nfp
            self.class_wise[class_label]['Nfn'] += Nfn
            self.class_wise[class_label]['Nref'] += Nref
            self.class_wise[class_label]['Nsys'] += Nsys

        return self

    def results(self):
        """Get results

        Outputs results in dict, format:

            {
                'overall':
                    {
                        'Pre':
                        'Rec':
                        'F':
                        'ER':
                        'S':
                        'D':
                        'I':
                    }
                'class_wise':
                    {
                        'office': {
                            'Pre':
                            'Rec':
                            'F':
                            'ER':
                            'D':
                            'I':
                            'Nref':
                            'Nsys':
                            'Ntp':
                            'Nfn':
                            'Nfp':
                        },
                    }
                'class_wise_average':
                    {
                        'F':
                        'ER':
                    }
            }

        Parameters
        ----------
        nothing

        Returns
        -------
        results : dict
            Results dict

        """

        results = {'overall': {},
                   'class_wise': {},
                   'class_wise_average': {},
                   }

        # Overall metrics
        results['overall']['Pre'] = self.overall['Ntp'] / (self.overall['Nsys'] + self.eps)
        results['overall']['Rec'] = self.overall['Ntp'] / self.overall['Nref']
        results['overall']['F'] = 2 * ((results['overall']['Pre'] * results['overall']['Rec']) / (results['overall']['Pre'] + results['overall']['Rec'] + self.eps))

        results['overall']['ER'] = self.overall['ER'] / self.overall['Nref']
        results['overall']['S'] = self.overall['S'] / self.overall['Nref']
        results['overall']['D'] = self.overall['D'] / self.overall['Nref']
        results['overall']['I'] = self.overall['I'] / self.overall['Nref']

        # Class-wise metrics
        class_wise_F = []
        class_wise_ER = []
        for class_id, class_label in enumerate(self.class_list):
            if class_label not in results['class_wise']:
                results['class_wise'][class_label] = {}
            results['class_wise'][class_label]['Pre'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nsys'] + self.eps)
            results['class_wise'][class_label]['Rec'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nref'] + self.eps)
            results['class_wise'][class_label]['F'] = 2 * ((results['class_wise'][class_label]['Pre'] * results['class_wise'][class_label]['Rec']) / (results['class_wise'][class_label]['Pre'] + results['class_wise'][class_label]['Rec'] + self.eps))

            results['class_wise'][class_label]['ER'] = (self.class_wise[class_label]['Nfn'] + self.class_wise[class_label]['Nfp']) / (self.class_wise[class_label]['Nref'] + self.eps)
            results['class_wise'][class_label]['D'] = self.class_wise[class_label]['Nfn'] / (self.class_wise[class_label]['Nref'] + self.eps)
            results['class_wise'][class_label]['I'] = self.class_wise[class_label]['Nfp'] / (self.class_wise[class_label]['Nref'] + self.eps)

            results['class_wise'][class_label]['Nref'] = self.class_wise[class_label]['Nref']
            results['class_wise'][class_label]['Nsys'] = self.class_wise[class_label]['Nsys']
            results['class_wise'][class_label]['Ntp'] = self.class_wise[class_label]['Ntp']
            results['class_wise'][class_label]['Nfn'] = self.class_wise[class_label]['Nfn']
            results['class_wise'][class_label]['Nfp'] = self.class_wise[class_label]['Nfp']

            class_wise_F.append(results['class_wise'][class_label]['F'])
            class_wise_ER.append(results['class_wise'][class_label]['ER'])

        results['class_wise_average']['F'] = numpy.mean(class_wise_F)
        results['class_wise_average']['ER'] = numpy.mean(class_wise_ER)

        return results

class DCASE2016_EventDetection_EventBasedMetrics(EventDetectionMetrics):
    """DCASE2016 Event based metrics for sound event detection

    Supported metrics:
    - Overall
        - Error rate (ER), Substitutions (S), Insertions (I), Deletions (D)
        - F-score (F1)
    - Class-wise
        - Error rate (ER), Insertions (I), Deletions (D)
        - F-score (F1)

    Examples
    --------

    >>> overall_metrics_per_scene = {}
    >>> for scene_id, scene_label in enumerate(dataset.scene_labels):
    >>>     dcase2016_event_based_metric = DCASE2016_EventDetection_EventBasedMetrics(class_list=dataset.event_labels(scene_label=scene_label))
    >>>     for fold in dataset.folds(mode=dataset_evaluation_mode):
    >>>         results = []
    >>>         result_filename = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)
    >>>
    >>>         if os.path.isfile(result_filename):
    >>>             with open(result_filename, 'rt') as f:
    >>>                 for row in csv.reader(f, delimiter='\t'):
    >>>                     results.append(row)
    >>>
    >>>         for file_id, item in enumerate(dataset.test(fold,scene_label=scene_label)):
    >>>             current_file_results = []
    >>>             for result_line in results:
    >>>                 if result_line[0] == dataset.absolute_to_relative(item['file']):
    >>>                     current_file_results.append(
    >>>                         {'file': result_line[0],
    >>>                          'event_onset': float(result_line[1]),
    >>>                          'event_offset': float(result_line[2]),
    >>>                          'event_label': result_line[3]
    >>>                          }
    >>>                     )
    >>>             meta = dataset.file_meta(dataset.absolute_to_relative(item['file']))
    >>>         dcase2016_event_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)
    >>> overall_metrics_per_scene[scene_label]['event_based_metrics'] = dcase2016_event_based_metric.results()

    """

    def __init__(self, class_list, time_resolution=1.0, t_collar=0.2):
        """__init__ method.

        Parameters
        ----------
        class_list : list
            List of class labels to be evaluated.

        time_resolution : float > 0
            Time resolution used when converting event into event roll.
            (Default value = 1.0)

        t_collar : float > 0
            Time collar for event onset and offset condition
            (Default value = 0.2)

        """

        self.time_resolution = time_resolution
        self.t_collar = t_collar

        self.overall = {
            'Nref': 0.0,
            'Nsys': 0.0,
            'Nsubs': 0.0,
            'Ntp': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
        }
        self.class_wise = {}

        for class_label in class_list:
            self.class_wise[class_label] = {
                'Nref': 0.0,
                'Nsys': 0.0,
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
            }

        EventDetectionMetrics.__init__(self, class_list=class_list)

    def __enter__(self):
        # Initialize class and return it
        return self

    def __exit__(self, type, value, traceback):
        # Finalize evaluation and return results
        return self.results()

    def evaluate(self, annotated_ground_truth, system_output):
        """Evaluate system output and annotated ground truth pair.

        Use results method to get results.

        Parameters
        ----------
        annotated_ground_truth : numpy.array
            Ground truth array, list of scene labels

        system_output : numpy.array
            System output array, list of scene labels

        Returns
        -------
        nothing

        """

        # Overall metrics

        # Total number of detected and reference events
        Nsys = len(system_output)
        Nref = len(annotated_ground_truth)

        sys_correct = numpy.zeros(Nsys, dtype=bool)
        ref_correct = numpy.zeros(Nref, dtype=bool)

        # Number of correctly transcribed events, onset/offset within a t_collar range
        for j in range(0, len(annotated_ground_truth)):
            for i in range(0, len(system_output)):
                label_condition = annotated_ground_truth[j]['event_label'] == system_output[i]['event_label']
                onset_condition = self.onset_condition(annotated_event=annotated_ground_truth[j],
                                                       system_event=system_output[i],
                                                       t_collar=self.t_collar)

                offset_condition = self.offset_condition(annotated_event=annotated_ground_truth[j],
                                                         system_event=system_output[i],
                                                         t_collar=self.t_collar)

                if label_condition and onset_condition and offset_condition:
                    ref_correct[j] = True
                    sys_correct[i] = True
                    break

        Ntp = numpy.sum(sys_correct)

        sys_leftover = numpy.nonzero(numpy.negative(sys_correct))[0]
        ref_leftover = numpy.nonzero(numpy.negative(ref_correct))[0]

        # Substitutions
        Nsubs = 0
        for j in ref_leftover:
            for i in sys_leftover:
                onset_condition = self.onset_condition(annotated_event=annotated_ground_truth[j],
                                                       system_event=system_output[i],
                                                       t_collar=self.t_collar)

                offset_condition = self.offset_condition(annotated_event=annotated_ground_truth[j],
                                                         system_event=system_output[i],
                                                         t_collar=self.t_collar)

                if onset_condition and offset_condition:
                    Nsubs += 1
                    break

        Nfp = Nsys - Ntp - Nsubs
        Nfn = Nref - Ntp - Nsubs

        self.overall['Nref'] += Nref
        self.overall['Nsys'] += Nsys
        self.overall['Ntp'] += Ntp
        self.overall['Nsubs'] += Nsubs
        self.overall['Nfp'] += Nfp
        self.overall['Nfn'] += Nfn

        # Class-wise metrics
        for class_id, class_label in enumerate(self.class_list):
            Nref = 0.0
            Nsys = 0.0
            Ntp = 0.0

            # Count event frequencies in the ground truth
            for i in range(0, len(annotated_ground_truth)):
                if annotated_ground_truth[i]['event_label'] == class_label:
                    Nref += 1

            # Count event frequencies in the system output
            for i in range(0, len(system_output)):
                if system_output[i]['event_label'] == class_label:
                    Nsys += 1

            for j in range(0, len(annotated_ground_truth)):
                for i in range(0, len(system_output)):
                    if annotated_ground_truth[j]['event_label'] == class_label and system_output[i]['event_label'] == class_label:
                        onset_condition = self.onset_condition(annotated_event=annotated_ground_truth[j],
                                                               system_event=system_output[i],
                                                               t_collar=self.t_collar)

                        offset_condition = self.offset_condition(annotated_event=annotated_ground_truth[j],
                                                                 system_event=system_output[i],
                                                                 t_collar=self.t_collar)

                        if onset_condition and offset_condition:
                            Ntp += 1
                            break

            Nfp = Nsys - Ntp
            Nfn = Nref - Ntp

            self.class_wise[class_label]['Nref'] += Nref
            self.class_wise[class_label]['Nsys'] += Nsys

            self.class_wise[class_label]['Ntp'] += Ntp
            self.class_wise[class_label]['Nfp'] += Nfp
            self.class_wise[class_label]['Nfn'] += Nfn

    def onset_condition(self, annotated_event, system_event, t_collar=0.200):
        """Onset condition, checked does the event pair fulfill condition

        Condition:

        - event onsets are within t_collar each other

        Parameters
        ----------
        annotated_event : dict
            Event dict

        system_event : dict
            Event dict

        t_collar : float > 0
            Defines how close event onsets have to be in order to be considered match. In seconds.
            (Default value = 0.2)

        Returns
        -------
        result : bool
            Condition result

        """

        return math.fabs(annotated_event['event_onset'] - system_event['event_onset']) <= t_collar

    def offset_condition(self, annotated_event, system_event, t_collar=0.200, percentage_of_length=0.5):
        """Offset condition, checking does the event pair fulfill condition

        Condition:

        - event offsets are within t_collar each other
        or
        - system event offset is within the percentage_of_length*annotated event_length

        Parameters
        ----------
        annotated_event : dict
            Event dict

        system_event : dict
            Event dict

        t_collar : float > 0
            Defines how close event onsets have to be in order to be considered match. In seconds.
            (Default value = 0.2)

        percentage_of_length : float [0-1]

        Returns
        -------
        result : bool
            Condition result

        """
        annotated_length = annotated_event['event_offset'] - annotated_event['event_onset']
        return math.fabs(annotated_event['event_offset'] - system_event['event_offset']) <= max(t_collar, percentage_of_length * annotated_length)

    def results(self):
        """Get results

        Outputs results in dict, format:

            {
                'overall':
                    {
                        'Pre':
                        'Rec':
                        'F':
                        'ER':
                        'S':
                        'D':
                        'I':
                    }
                'class_wise':
                    {
                        'office': {
                            'Pre':
                            'Rec':
                            'F':
                            'ER':
                            'D':
                            'I':
                            'Nref':
                            'Nsys':
                            'Ntp':
                            'Nfn':
                            'Nfp':
                        },
                    }
                'class_wise_average':
                    {
                        'F':
                        'ER':
                    }
            }

        Parameters
        ----------
        nothing

        Returns
        -------
        results : dict
            Results dict

        """

        results = {
            'overall': {},
            'class_wise': {},
            'class_wise_average': {},
        }

        # Overall metrics
        results['overall']['Pre'] = self.overall['Ntp'] / (self.overall['Nsys'] + self.eps)
        results['overall']['Rec'] = self.overall['Ntp'] / self.overall['Nref']
        results['overall']['F'] = 2 * ((results['overall']['Pre'] * results['overall']['Rec']) / (results['overall']['Pre'] + results['overall']['Rec'] + self.eps))

        results['overall']['ER'] = (self.overall['Nfn'] + self.overall['Nfp'] + self.overall['Nsubs']) / self.overall['Nref']
        results['overall']['S'] = self.overall['Nsubs'] / self.overall['Nref']
        results['overall']['D'] = self.overall['Nfn'] / self.overall['Nref']
        results['overall']['I'] = self.overall['Nfp'] / self.overall['Nref']

        # Class-wise metrics
        class_wise_F = []
        class_wise_ER = []

        for class_label in self.class_list:
            if class_label not in results['class_wise']:
                results['class_wise'][class_label] = {}

            results['class_wise'][class_label]['Pre'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nsys'] + self.eps)
            results['class_wise'][class_label]['Rec'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nref'] + self.eps)
            results['class_wise'][class_label]['F'] = 2 * ((results['class_wise'][class_label]['Pre'] * results['class_wise'][class_label]['Rec']) / (results['class_wise'][class_label]['Pre'] + results['class_wise'][class_label]['Rec'] + self.eps))

            results['class_wise'][class_label]['ER'] = (self.class_wise[class_label]['Nfn']+self.class_wise[class_label]['Nfp']) / (self.class_wise[class_label]['Nref'] + self.eps)
            results['class_wise'][class_label]['D'] = self.class_wise[class_label]['Nfn'] / (self.class_wise[class_label]['Nref'] + self.eps)
            results['class_wise'][class_label]['I'] = self.class_wise[class_label]['Nfp'] / (self.class_wise[class_label]['Nref'] + self.eps)

            results['class_wise'][class_label]['Nref'] = self.class_wise[class_label]['Nref']
            results['class_wise'][class_label]['Nsys'] = self.class_wise[class_label]['Nsys']
            results['class_wise'][class_label]['Ntp'] = self.class_wise[class_label]['Ntp']
            results['class_wise'][class_label]['Nfn'] = self.class_wise[class_label]['Nfn']
            results['class_wise'][class_label]['Nfp'] = self.class_wise[class_label]['Nfp']

            class_wise_F.append(results['class_wise'][class_label]['F'])
            class_wise_ER.append(results['class_wise'][class_label]['ER'])

        # Class-wise average
        results['class_wise_average']['F'] = numpy.mean(class_wise_F)
        results['class_wise_average']['ER'] = numpy.mean(class_wise_ER)

        return results

class DCASE2013_EventDetection_Metrics(EventDetectionMetrics):
    """Lecagy DCASE2013 metrics, converted from the provided Matlab implementation

    Supported metrics:
    - Frame based
        - F-score (F)
        - AEER
    - Event based
        - Onset
            - F-Score (F)
            - AEER
        - Onset-offset
            - F-Score (F)
            - AEER
    - Class based
        - Onset
            - F-Score (F)
            - AEER
        - Onset-offset
            - F-Score (F)
            - AEER
    """

    #

    def frame_based(self, annotated_ground_truth, system_output, resolution=0.01):
        # Convert event list into frame-based representation
        system_event_roll = self.list_to_roll(data=system_output, time_resolution=resolution)
        annotated_event_roll = self.list_to_roll(data=annotated_ground_truth, time_resolution=resolution)

        # Fix durations of both event_rolls to be equal
        if annotated_event_roll.shape[0] > system_event_roll.shape[0]:
            padding = numpy.zeros((annotated_event_roll.shape[0] - system_event_roll.shape[0], len(self.class_list)))
            system_event_roll = numpy.vstack((system_event_roll, padding))

        if system_event_roll.shape[0] > annotated_event_roll.shape[0]:
            padding = numpy.zeros((system_event_roll.shape[0] - annotated_event_roll.shape[0], len(self.class_list)))
            annotated_event_roll = numpy.vstack((annotated_event_roll, padding))

        # Compute frame-based metrics
        Nref = sum(sum(annotated_event_roll))
        Ntot = sum(sum(system_event_roll))
        Ntp = sum(sum(system_event_roll + annotated_event_roll > 1))
        Nfp = sum(sum(system_event_roll - annotated_event_roll > 0))
        Nfn = sum(sum(annotated_event_roll - system_event_roll > 0))
        Nsubs = min(Nfp, Nfn)

        eps = numpy.spacing(1)

        results = dict()
        results['Rec'] = Ntp / (Nref + eps)
        results['Pre'] = Ntp / (Ntot + eps)
        results['F'] = 2 * ((results['Pre'] * results['Rec']) / (results['Pre'] + results['Rec'] + eps))
        results['AEER'] = (Nfn + Nfp + Nsubs) / (Nref + eps)

        return results

    def event_based(self, annotated_ground_truth, system_output):
        # Event-based evaluation for event detection task
        # outputFile: the output of the event detection system
        # GTFile: the ground truth list of events

        # Total number of detected and reference events
        Ntot = len(system_output)
        Nref = len(annotated_ground_truth)

        # Number of correctly transcribed events, onset within a +/-100 ms range
        Ncorr = 0
        NcorrOff = 0
        for j in range(0, len(annotated_ground_truth)):
            for i in range(0, len(system_output)):
                if annotated_ground_truth[j]['event_label'] == system_output[i]['event_label'] and (math.fabs(annotated_ground_truth[j]['event_onset'] - system_output[i]['event_onset']) <= 0.1):
                    Ncorr += 1

                    # If offset within a +/-100 ms range or within 50% of ground-truth event's duration
                    if math.fabs(annotated_ground_truth[j]['event_offset'] - system_output[i]['event_offset']) <= max(0.1, 0.5 * (annotated_ground_truth[j]['event_offset'] - annotated_ground_truth[j]['event_onset'])):
                        NcorrOff += 1

                    break  # In order to not evaluate duplicates

        # Compute onset-only event-based metrics
        eps = numpy.spacing(1)
        results = {
            'onset': {},
            'onset-offset': {},
        }

        Nfp = Ntot - Ncorr
        Nfn = Nref - Ncorr
        Nsubs = min(Nfp, Nfn)
        results['onset']['Rec'] = Ncorr / (Nref + eps)
        results['onset']['Pre'] = Ncorr / (Ntot + eps)
        results['onset']['F'] = 2 * (
            (results['onset']['Pre'] * results['onset']['Rec']) / (
                results['onset']['Pre'] + results['onset']['Rec'] + eps))
        results['onset']['AEER'] = (Nfn + Nfp + Nsubs) / (Nref + eps)

        # Compute onset-offset event-based metrics
        NfpOff = Ntot - NcorrOff
        NfnOff = Nref - NcorrOff
        NsubsOff = min(NfpOff, NfnOff)
        results['onset-offset']['Rec'] = NcorrOff / (Nref + eps)
        results['onset-offset']['Pre'] = NcorrOff / (Ntot + eps)
        results['onset-offset']['F'] = 2 * ((results['onset-offset']['Pre'] * results['onset-offset']['Rec']) / (
            results['onset-offset']['Pre'] + results['onset-offset']['Rec'] + eps))
        results['onset-offset']['AEER'] = (NfnOff + NfpOff + NsubsOff) / (Nref + eps)

        return results

    def class_based(self, annotated_ground_truth, system_output):
        # Class-wise event-based evaluation for event detection task
        # outputFile: the output of the event detection system
        # GTFile: the ground truth list of events

        # Total number of detected and reference events per class
        Ntot = numpy.zeros((len(self.class_list), 1))
        for event in system_output:
            pos = self.class_list.index(event['event_label'])
            Ntot[pos] += 1

        Nref = numpy.zeros((len(self.class_list), 1))
        for event in annotated_ground_truth:
            pos = self.class_list.index(event['event_label'])
            Nref[pos] += 1

        I = (Nref > 0).nonzero()[0]  # index for classes present in ground-truth

        # Number of correctly transcribed events per class, onset within a +/-100 ms range
        Ncorr = numpy.zeros((len(self.class_list), 1))
        NcorrOff = numpy.zeros((len(self.class_list), 1))

        for j in range(0, len(annotated_ground_truth)):
            for i in range(0, len(system_output)):
                if annotated_ground_truth[j]['event_label'] == system_output[i]['event_label'] and (
                            math.fabs(
                                    annotated_ground_truth[j]['event_onset'] - system_output[i]['event_onset']) <= 0.1):
                    pos = self.class_list.index(system_output[i]['event_label'])
                    Ncorr[pos] += 1

                    # If offset within a +/-100 ms range or within 50% of ground-truth event's duration
                    if math.fabs(annotated_ground_truth[j]['event_offset'] - system_output[i]['event_offset']) <= max(
                            0.1, 0.5 * (
                                        annotated_ground_truth[j]['event_offset'] - annotated_ground_truth[j][
                                        'event_onset'])):
                        pos = self.class_list.index(system_output[i]['event_label'])
                        NcorrOff[pos] += 1

                    break  # In order to not evaluate duplicates

        # Compute onset-only class-wise event-based metrics
        eps = numpy.spacing(1)
        results = {
            'onset': {},
            'onset-offset': {},
        }

        Nfp = Ntot - Ncorr
        Nfn = Nref - Ncorr
        Nsubs = numpy.minimum(Nfp, Nfn)
        tempRec = Ncorr[I] / (Nref[I] + eps)
        tempPre = Ncorr[I] / (Ntot[I] + eps)
        results['onset']['Rec'] = numpy.mean(tempRec)
        results['onset']['Pre'] = numpy.mean(tempPre)
        tempF = 2 * ((tempPre * tempRec) / (tempPre + tempRec + eps))
        results['onset']['F'] = numpy.mean(tempF)
        tempAEER = (Nfn[I] + Nfp[I] + Nsubs[I]) / (Nref[I] + eps)
        results['onset']['AEER'] = numpy.mean(tempAEER)

        # Compute onset-offset class-wise event-based metrics
        NfpOff = Ntot - NcorrOff
        NfnOff = Nref - NcorrOff
        NsubsOff = numpy.minimum(NfpOff, NfnOff)
        tempRecOff = NcorrOff[I] / (Nref[I] + eps)
        tempPreOff = NcorrOff[I] / (Ntot[I] + eps)
        results['onset-offset']['Rec'] = numpy.mean(tempRecOff)
        results['onset-offset']['Pre'] = numpy.mean(tempPreOff)
        tempFOff = 2 * ((tempPreOff * tempRecOff) / (tempPreOff + tempRecOff + eps))
        results['onset-offset']['F'] = numpy.mean(tempFOff)
        tempAEEROff = (NfnOff[I] + NfpOff[I] + NsubsOff[I]) / (Nref[I] + eps)
        results['onset-offset']['AEER'] = numpy.mean(tempAEEROff)

        return results

def main(argv):
    # Examples to show usage and required data structures
    class_list = ['class1', 'class2', 'class3']
    system_output = [
        {
            'event_label': 'class1',
            'event_onset': 0.1,
            'event_offset': 1.0
        },
        {
            'event_label': 'class2',
            'event_onset': 4.1,
            'event_offset': 4.7
        },
        {
            'event_label': 'class3',
            'event_onset': 5.5,
            'event_offset': 6.7
        }
    ]
    annotated_groundtruth = [
        {
            'event_label': 'class1',
            'event_onset': 0.1,
            'event_offset': 1.0
        },
        {
            'event_label': 'class2',
            'event_onset': 4.2,
            'event_offset': 5.4
        },
        {
            'event_label': 'class3',
            'event_onset': 5.5,
            'event_offset': 6.7
        }
    ]
    dcase2013metric = DCASE2013_EventDetection_Metrics(class_list=class_list)

    print 'DCASE2013'
    print 'Frame-based:', dcase2013metric.frame_based(system_output=system_output,
                                                      annotated_ground_truth=annotated_groundtruth)
    print 'Event-based:', dcase2013metric.event_based(system_output=system_output,
                                                      annotated_ground_truth=annotated_groundtruth)
    print 'Class-based:', dcase2013metric.class_based(system_output=system_output,
                                                      annotated_ground_truth=annotated_groundtruth)

    dcase2016_metric = DCASE2016_EventDetection_SegmentBasedMetrics(class_list=class_list)
    print 'DCASE2016'
    print dcase2016_metric.evaluate(system_output=system_output, annotated_ground_truth=annotated_groundtruth).results()

if __name__ == "__main__":
    sys.exit(main(sys.argv))


"""Testing for the boost module (sklearn.ensemble.boost)."""

import numpy as np
from sklearn.utils.testing import assert_array_equal, assert_array_less
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_equal, assert_true
from sklearn.utils.testing import assert_raises, assert_raises_regexp

from sklearn.cross_validation import train_test_split
from sklearn.grid_search import GridSearchCV
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import weight_boosting
from scipy.sparse import csc_matrix
from scipy.sparse import csr_matrix
from scipy.sparse import coo_matrix
from scipy.sparse import dok_matrix
from scipy.sparse import lil_matrix
from sklearn.svm import SVC, SVR
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.utils import shuffle
from sklearn import datasets

# Common random state
rng = np.random.RandomState(0)

# Toy sample
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
y_class = ["foo", "foo", "foo", 1, 1, 1]    # test string class labels
y_regr = [-1, -1, -1, 1, 1, 1]
T = [[-1, -1], [2, 2], [3, 2]]
y_t_class = ["foo", 1, 1]
y_t_regr = [-1, 1, 1]

# Load the iris dataset and randomly permute it
iris = datasets.load_iris()
perm = rng.permutation(iris.target.size)
iris.data, iris.target = shuffle(iris.data, iris.target, random_state=rng)

# Load the boston dataset and randomly permute it
boston = datasets.load_boston()
boston.data, boston.target = shuffle(boston.data, boston.target,
                                     random_state=rng)

def test_samme_proba():
    # Test the `_samme_proba` helper function.

    # Define some example (bad) `predict_proba` output.
    probs = np.array([[1, 1e-6, 0],
                      [0.19, 0.6, 0.2],
                      [-999, 0.51, 0.5],
                      [1e-6, 1, 1e-9]])
    probs /= np.abs(probs.sum(axis=1))[:, np.newaxis]

    # _samme_proba calls estimator.predict_proba.
    # Make a mock object so I can control what gets returned.
    class MockEstimator(object):
        def predict_proba(self, X):
            assert_array_equal(X.shape, probs.shape)
            return probs
    mock = MockEstimator()

    samme_proba = weight_boosting._samme_proba(mock, 3, np.ones_like(probs))

    assert_array_equal(samme_proba.shape, probs.shape)
    assert_true(np.isfinite(samme_proba).all())

    # Make sure that the correct elements come out as smallest --
    # `_samme_proba` should preserve the ordering in each example.
    assert_array_equal(np.argmin(samme_proba, axis=1), [2, 0, 0, 2])
    assert_array_equal(np.argmax(samme_proba, axis=1), [0, 1, 1, 1])

def test_classification_toy():
    # Check classification on a toy dataset.
    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg, random_state=0)
        clf.fit(X, y_class)
        assert_array_equal(clf.predict(T), y_t_class)
        assert_array_equal(np.unique(np.asarray(y_t_class)), clf.classes_)
        assert_equal(clf.predict_proba(T).shape, (len(T), 2))
        assert_equal(clf.decision_function(T).shape, (len(T),))

def test_regression_toy():
    # Check classification on a toy dataset.
    clf = AdaBoostRegressor(random_state=0)
    clf.fit(X, y_regr)
    assert_array_equal(clf.predict(T), y_t_regr)

def test_iris():
    # Check consistency on dataset iris.
    classes = np.unique(iris.target)
    clf_samme = prob_samme = None

    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg)
        clf.fit(iris.data, iris.target)

        assert_array_equal(classes, clf.classes_)
        proba = clf.predict_proba(iris.data)
        if alg == "SAMME":
            clf_samme = clf
            prob_samme = proba
        assert_equal(proba.shape[1], len(classes))
        assert_equal(clf.decision_function(iris.data).shape[1], len(classes))

        score = clf.score(iris.data, iris.target)
        assert score > 0.9, "Failed with algorithm %s and score = %f" % \
            (alg, score)

    # Somewhat hacky regression test: prior to
    # ae7adc880d624615a34bafdb1d75ef67051b8200,
    # predict_proba returned SAMME.R values for SAMME.
    clf_samme.algorithm = "SAMME.R"
    assert_array_less(0,
                      np.abs(clf_samme.predict_proba(iris.data) - prob_samme))

def test_boston():
    # Check consistency on dataset boston house prices.
    clf = AdaBoostRegressor(random_state=0)
    clf.fit(boston.data, boston.target)
    score = clf.score(boston.data, boston.target)
    assert score > 0.85

def test_staged_predict():
    # Check staged predictions.
    rng = np.random.RandomState(0)
    iris_weights = rng.randint(10, size=iris.target.shape)
    boston_weights = rng.randint(10, size=boston.target.shape)

    # AdaBoost classification
    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg, n_estimators=10)
        clf.fit(iris.data, iris.target, sample_weight=iris_weights)

        predictions = clf.predict(iris.data)
        staged_predictions = [p for p in clf.staged_predict(iris.data)]
        proba = clf.predict_proba(iris.data)
        staged_probas = [p for p in clf.staged_predict_proba(iris.data)]
        score = clf.score(iris.data, iris.target, sample_weight=iris_weights)
        staged_scores = [
            s for s in clf.staged_score(
                iris.data, iris.target, sample_weight=iris_weights)]

        assert_equal(len(staged_predictions), 10)
        assert_array_almost_equal(predictions, staged_predictions[-1])
        assert_equal(len(staged_probas), 10)
        assert_array_almost_equal(proba, staged_probas[-1])
        assert_equal(len(staged_scores), 10)
        assert_array_almost_equal(score, staged_scores[-1])

    # AdaBoost regression
    clf = AdaBoostRegressor(n_estimators=10, random_state=0)
    clf.fit(boston.data, boston.target, sample_weight=boston_weights)

    predictions = clf.predict(boston.data)
    staged_predictions = [p for p in clf.staged_predict(boston.data)]
    score = clf.score(boston.data, boston.target, sample_weight=boston_weights)
    staged_scores = [
        s for s in clf.staged_score(
            boston.data, boston.target, sample_weight=boston_weights)]

    assert_equal(len(staged_predictions), 10)
    assert_array_almost_equal(predictions, staged_predictions[-1])
    assert_equal(len(staged_scores), 10)
    assert_array_almost_equal(score, staged_scores[-1])

def test_gridsearch():
    # Check that base trees can be grid-searched.
    # AdaBoost classification
    boost = AdaBoostClassifier(base_estimator=DecisionTreeClassifier())
    parameters = {'n_estimators': (1, 2),
                  'base_estimator__max_depth': (1, 2),
                  'algorithm': ('SAMME', 'SAMME.R')}
    clf = GridSearchCV(boost, parameters)
    clf.fit(iris.data, iris.target)

    # AdaBoost regression
    boost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(),
                              random_state=0)
    parameters = {'n_estimators': (1, 2),
                  'base_estimator__max_depth': (1, 2)}
    clf = GridSearchCV(boost, parameters)
    clf.fit(boston.data, boston.target)

def test_pickle():
    # Check pickability.
    import pickle

    # Adaboost classifier
    for alg in ['SAMME', 'SAMME.R']:
        obj = AdaBoostClassifier(algorithm=alg)
        obj.fit(iris.data, iris.target)
        score = obj.score(iris.data, iris.target)
        s = pickle.dumps(obj)

        obj2 = pickle.loads(s)
        assert_equal(type(obj2), obj.__class__)
        score2 = obj2.score(iris.data, iris.target)
        assert_equal(score, score2)

    # Adaboost regressor
    obj = AdaBoostRegressor(random_state=0)
    obj.fit(boston.data, boston.target)
    score = obj.score(boston.data, boston.target)
    s = pickle.dumps(obj)

    obj2 = pickle.loads(s)
    assert_equal(type(obj2), obj.__class__)
    score2 = obj2.score(boston.data, boston.target)
    assert_equal(score, score2)

def test_importances():
    # Check variable importances.
    X, y = datasets.make_classification(n_samples=2000,
                                        n_features=10,
                                        n_informative=3,
                                        n_redundant=0,
                                        n_repeated=0,
                                        shuffle=False,
                                        random_state=1)

    for alg in ['SAMME', 'SAMME.R']:
        clf = AdaBoostClassifier(algorithm=alg)

        clf.fit(X, y)
        importances = clf.feature_importances_

        assert_equal(importances.shape[0], 10)
        assert_equal((importances[:3, np.newaxis] >= importances[3:]).all(),
                     True)

def test_error():
    # Test that it gives proper exception on deficient input.
    assert_raises(ValueError,
                  AdaBoostClassifier(learning_rate=-1).fit,
                  X, y_class)

    assert_raises(ValueError,
                  AdaBoostClassifier(algorithm="foo").fit,
                  X, y_class)

    assert_raises(ValueError,
                  AdaBoostClassifier().fit,
                  X, y_class, sample_weight=np.asarray([-1]))

def test_base_estimator():
    # Test different base estimators.
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.svm import SVC

    # XXX doesn't work with y_class because RF doesn't support classes_
    # Shouldn't AdaBoost run a LabelBinarizer?
    clf = AdaBoostClassifier(RandomForestClassifier())
    clf.fit(X, y_regr)

    clf = AdaBoostClassifier(SVC(), algorithm="SAMME")
    clf.fit(X, y_class)

    from sklearn.ensemble import RandomForestRegressor
    from sklearn.svm import SVR

    clf = AdaBoostRegressor(RandomForestRegressor(), random_state=0)
    clf.fit(X, y_regr)

    clf = AdaBoostRegressor(SVR(), random_state=0)
    clf.fit(X, y_regr)

    # Check that an empty discrete ensemble fails in fit, not predict.
    X_fail = [[1, 1], [1, 1], [1, 1], [1, 1]]
    y_fail = ["foo", "bar", 1, 2]
    clf = AdaBoostClassifier(SVC(), algorithm="SAMME")
    assert_raises_regexp(ValueError, "worse than random",
                         clf.fit, X_fail, y_fail)

def test_sample_weight_missing():
    from sklearn.linear_model import LinearRegression
    from sklearn.cluster import KMeans

    clf = AdaBoostClassifier(LinearRegression(), algorithm="SAMME")
    assert_raises(ValueError, clf.fit, X, y_regr)

    clf = AdaBoostRegressor(LinearRegression())
    assert_raises(ValueError, clf.fit, X, y_regr)

    clf = AdaBoostClassifier(KMeans(), algorithm="SAMME")
    assert_raises(ValueError, clf.fit, X, y_regr)

    clf = AdaBoostRegressor(KMeans())
    assert_raises(ValueError, clf.fit, X, y_regr)

def test_sparse_classification():
    # Check classification with sparse input.

    class CustomSVC(SVC):
        """SVC variant that records the nature of the training set."""

        def fit(self, X, y, sample_weight=None):
            """Modification on fit caries data type for later verification."""
            super(CustomSVC, self).fit(X, y, sample_weight=sample_weight)
            self.data_type_ = type(X)
            return self

    X, y = datasets.make_multilabel_classification(n_classes=1, n_samples=15,
                                                   n_features=5,
                                                   random_state=42)
    # Flatten y to a 1d array
    y = np.ravel(y)

    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    for sparse_format in [csc_matrix, csr_matrix, lil_matrix, coo_matrix,
                          dok_matrix]:
        X_train_sparse = sparse_format(X_train)
        X_test_sparse = sparse_format(X_test)

        # Trained on sparse format
        sparse_classifier = AdaBoostClassifier(
            base_estimator=CustomSVC(probability=True),
            random_state=1,
            algorithm="SAMME"
        ).fit(X_train_sparse, y_train)

        # Trained on dense format
        dense_classifier = AdaBoostClassifier(
            base_estimator=CustomSVC(probability=True),
            random_state=1,
            algorithm="SAMME"
        ).fit(X_train, y_train)

        # predict
        sparse_results = sparse_classifier.predict(X_test_sparse)
        dense_results = dense_classifier.predict(X_test)
        assert_array_equal(sparse_results, dense_results)

        # decision_function
        sparse_results = sparse_classifier.decision_function(X_test_sparse)
        dense_results = dense_classifier.decision_function(X_test)
        assert_array_equal(sparse_results, dense_results)

        # predict_log_proba
        sparse_results = sparse_classifier.predict_log_proba(X_test_sparse)
        dense_results = dense_classifier.predict_log_proba(X_test)
        assert_array_equal(sparse_results, dense_results)

        # predict_proba
        sparse_results = sparse_classifier.predict_proba(X_test_sparse)
        dense_results = dense_classifier.predict_proba(X_test)
        assert_array_equal(sparse_results, dense_results)

        # score
        sparse_results = sparse_classifier.score(X_test_sparse, y_test)
        dense_results = dense_classifier.score(X_test, y_test)
        assert_array_equal(sparse_results, dense_results)

        # staged_decision_function
        sparse_results = sparse_classifier.staged_decision_function(
            X_test_sparse)
        dense_results = dense_classifier.staged_decision_function(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # staged_predict
        sparse_results = sparse_classifier.staged_predict(X_test_sparse)
        dense_results = dense_classifier.staged_predict(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # staged_predict_proba
        sparse_results = sparse_classifier.staged_predict_proba(X_test_sparse)
        dense_results = dense_classifier.staged_predict_proba(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # staged_score
        sparse_results = sparse_classifier.staged_score(X_test_sparse,
                                                        y_test)
        dense_results = dense_classifier.staged_score(X_test, y_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        # Verify sparsity of data is maintained during training
        types = [i.data_type_ for i in sparse_classifier.estimators_]

        assert all([(t == csc_matrix or t == csr_matrix)
                   for t in types])

def test_sparse_regression():
    # Check regression with sparse input.

    class CustomSVR(SVR):
        """SVR variant that records the nature of the training set."""

        def fit(self, X, y, sample_weight=None):
            """Modification on fit caries data type for later verification."""
            super(CustomSVR, self).fit(X, y, sample_weight=sample_weight)
            self.data_type_ = type(X)
            return self

    X, y = datasets.make_regression(n_samples=15, n_features=50, n_targets=1,
                                    random_state=42)

    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    for sparse_format in [csc_matrix, csr_matrix, lil_matrix, coo_matrix,
                          dok_matrix]:
        X_train_sparse = sparse_format(X_train)
        X_test_sparse = sparse_format(X_test)

        # Trained on sparse format
        sparse_classifier = AdaBoostRegressor(
            base_estimator=CustomSVR(),
            random_state=1
        ).fit(X_train_sparse, y_train)

        # Trained on dense format
        dense_classifier = dense_results = AdaBoostRegressor(
            base_estimator=CustomSVR(),
            random_state=1
        ).fit(X_train, y_train)

        # predict
        sparse_results = sparse_classifier.predict(X_test_sparse)
        dense_results = dense_classifier.predict(X_test)
        assert_array_equal(sparse_results, dense_results)

        # staged_predict
        sparse_results = sparse_classifier.staged_predict(X_test_sparse)
        dense_results = dense_classifier.staged_predict(X_test)
        for sprase_res, dense_res in zip(sparse_results, dense_results):
            assert_array_equal(sprase_res, dense_res)

        types = [i.data_type_ for i in sparse_classifier.estimators_]

        assert all([(t == csc_matrix or t == csr_matrix)
                   for t in types])


"""Utilities to evaluate the clustering performance of models

Functions named as *_score return a scalar value to maximize: the higher the
better.
"""

# Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Wei LI <kuantkid@gmail.com>
#          Diego Molla <dmolla-aliod@gmail.com>
# License: BSD 3 clause

from math import log

from scipy.misc import comb
from scipy.sparse import coo_matrix
import numpy as np

from .expected_mutual_info_fast import expected_mutual_information
from ...utils.fixes import bincount

def comb2(n):
    # the exact version is faster for k == 2: use it by default globally in
    # this module instead of the float approximate variant
    return comb(n, 2, exact=1)

def check_clusterings(labels_true, labels_pred):
    """Check that the two clusterings matching 1D integer arrays"""
    labels_true = np.asarray(labels_true)
    labels_pred = np.asarray(labels_pred)

    # input checks
    if labels_true.ndim != 1:
        raise ValueError(
            "labels_true must be 1D: shape is %r" % (labels_true.shape,))
    if labels_pred.ndim != 1:
        raise ValueError(
            "labels_pred must be 1D: shape is %r" % (labels_pred.shape,))
    if labels_true.shape != labels_pred.shape:
        raise ValueError(
            "labels_true and labels_pred must have same size, got %d and %d"
            % (labels_true.shape[0], labels_pred.shape[0]))
    return labels_true, labels_pred

def contingency_matrix(labels_true, labels_pred, eps=None):
    """Build a contengency matrix describing the relationship between labels.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        Ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        Cluster labels to evaluate

    eps: None or float
        If a float, that value is added to all values in the contingency
        matrix. This helps to stop NaN propagation.
        If ``None``, nothing is adjusted.

    Returns
    -------
    contingency: array, shape=[n_classes_true, n_classes_pred]
        Matrix :math:`C` such that :math:`C_{i, j}` is the number of samples in
        true class :math:`i` and in predicted class :math:`j`. If
        ``eps is None``, the dtype of this array will be integer. If ``eps`` is
        given, the dtype will be float.
    """
    classes, class_idx = np.unique(labels_true, return_inverse=True)
    clusters, cluster_idx = np.unique(labels_pred, return_inverse=True)
    n_classes = classes.shape[0]
    n_clusters = clusters.shape[0]
    # Using coo_matrix to accelerate simple histogram calculation,
    # i.e. bins are consecutive integers
    # Currently, coo_matrix is faster than histogram2d for simple cases
    contingency = coo_matrix((np.ones(class_idx.shape[0]),
                              (class_idx, cluster_idx)),
                             shape=(n_classes, n_clusters),
                             dtype=np.int).toarray()
    if eps is not None:
        # don't use += as contingency is integer
        contingency = contingency + eps
    return contingency

# clustering measures

def adjusted_rand_score(labels_true, labels_pred):
    """Rand index adjusted for chance

    The Rand Index computes a similarity measure between two clusterings
    by considering all pairs of samples and counting pairs that are
    assigned in the same or different clusters in the predicted and
    true clusterings.

    The raw RI score is then "adjusted for chance" into the ARI score
    using the following scheme::

        ARI = (RI - Expected_RI) / (max(RI) - Expected_RI)

    The adjusted Rand index is thus ensured to have a value close to
    0.0 for random labeling independently of the number of clusters and
    samples and exactly 1.0 when the clusterings are identical (up to
    a permutation).

    ARI is a symmetric measure::

        adjusted_rand_score(a, b) == adjusted_rand_score(b, a)

    Read more in the :ref:`User Guide <adjusted_rand_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        Ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        Cluster labels to evaluate

    Returns
    -------
    ari : float
       Similarity score between -1.0 and 1.0. Random labelings have an ARI
       close to 0.0. 1.0 stands for perfect match.

    Examples
    --------

    Perfectly maching labelings have a score of 1 even

      >>> from sklearn.metrics.cluster import adjusted_rand_score
      >>> adjusted_rand_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> adjusted_rand_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Labelings that assign all classes members to the same clusters
    are complete be not always pure, hence penalized::

      >>> adjusted_rand_score([0, 0, 1, 2], [0, 0, 1, 1])  # doctest: +ELLIPSIS
      0.57...

    ARI is symmetric, so labelings that have pure clusters with members
    coming from the same classes but unnecessary splits are penalized::

      >>> adjusted_rand_score([0, 0, 1, 1], [0, 0, 1, 2])  # doctest: +ELLIPSIS
      0.57...

    If classes members are completely split across different clusters, the
    assignment is totally incomplete, hence the ARI is very low::

      >>> adjusted_rand_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    References
    ----------

    .. [Hubert1985] `L. Hubert and P. Arabie, Comparing Partitions,
      Journal of Classification 1985`
      http://www.springerlink.com/content/x64124718341j1j0/

    .. [wk] http://en.wikipedia.org/wiki/Rand_index#Adjusted_Rand_index

    See also
    --------
    adjusted_mutual_info_score: Adjusted Mutual Information

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    n_samples = labels_true.shape[0]
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split;
    # or trivial clustering where each document is assigned a unique cluster.
    # These are perfect matches hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1
            or classes.shape[0] == clusters.shape[0] == 0
            or classes.shape[0] == clusters.shape[0] == len(labels_true)):
        return 1.0

    contingency = contingency_matrix(labels_true, labels_pred)

    # Compute the ARI using the contingency data
    sum_comb_c = sum(comb2(n_c) for n_c in contingency.sum(axis=1))
    sum_comb_k = sum(comb2(n_k) for n_k in contingency.sum(axis=0))

    sum_comb = sum(comb2(n_ij) for n_ij in contingency.flatten())
    prod_comb = (sum_comb_c * sum_comb_k) / float(comb(n_samples, 2))
    mean_comb = (sum_comb_k + sum_comb_c) / 2.
    return ((sum_comb - prod_comb) / (mean_comb - prod_comb))

def homogeneity_completeness_v_measure(labels_true, labels_pred):
    """Compute the homogeneity and completeness and V-Measure scores at once

    Those metrics are based on normalized conditional entropy measures of
    the clustering labeling to evaluate given the knowledge of a Ground
    Truth class labels of the same samples.

    A clustering result satisfies homogeneity if all of its clusters
    contain only data points which are members of a single class.

    A clustering result satisfies completeness if all the data points
    that are members of a given class are elements of the same cluster.

    Both scores have positive values between 0.0 and 1.0, larger values
    being desirable.

    Those 3 metrics are independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score values in any way.

    V-Measure is furthermore symmetric: swapping ``labels_true`` and
    ``label_pred`` will give the same score. This does not hold for
    homogeneity and completeness.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    homogeneity: float
       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling

    completeness: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    v_measure: float
        harmonic mean of the first two

    See also
    --------
    homogeneity_score
    completeness_score
    v_measure_score
    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)

    if len(labels_true) == 0:
        return 1.0, 1.0, 1.0

    entropy_C = entropy(labels_true)
    entropy_K = entropy(labels_pred)

    MI = mutual_info_score(labels_true, labels_pred)

    homogeneity = MI / (entropy_C) if entropy_C else 1.0
    completeness = MI / (entropy_K) if entropy_K else 1.0

    if homogeneity + completeness == 0.0:
        v_measure_score = 0.0
    else:
        v_measure_score = (2.0 * homogeneity * completeness
                           / (homogeneity + completeness))

    return homogeneity, completeness, v_measure_score

def homogeneity_score(labels_true, labels_pred):
    """Homogeneity metric of a cluster labeling given a ground truth

    A clustering result satisfies homogeneity if all of its clusters
    contain only data points which are members of a single class.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is not symmetric: switching ``label_true`` with ``label_pred``
    will return the :func:`completeness_score` which will be different in
    general.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    homogeneity: float
       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    completeness_score
    v_measure_score

    Examples
    --------

    Perfect labelings are homogeneous::

      >>> from sklearn.metrics.cluster import homogeneity_score
      >>> homogeneity_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Non-perfect labelings that further split classes into more clusters can be
    perfectly homogeneous::

      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 0, 1, 2]))
      ...                                                  # doctest: +ELLIPSIS
      1.0...
      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      1.0...

    Clusters that include samples from different classes do not make for an
    homogeneous labeling::

      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 1, 0, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...
      >>> print("%.6f" % homogeneity_score([0, 0, 1, 1], [0, 0, 0, 0]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred)[0]

def completeness_score(labels_true, labels_pred):
    """Completeness metric of a cluster labeling given a ground truth

    A clustering result satisfies completeness if all the data points
    that are members of a given class are elements of the same cluster.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is not symmetric: switching ``label_true`` with ``label_pred``
    will return the :func:`homogeneity_score` which will be different in
    general.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    completeness: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    homogeneity_score
    v_measure_score

    Examples
    --------

    Perfect labelings are complete::

      >>> from sklearn.metrics.cluster import completeness_score
      >>> completeness_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Non-perfect labelings that assign all classes members to the same clusters
    are still complete::

      >>> print(completeness_score([0, 0, 1, 1], [0, 0, 0, 0]))
      1.0
      >>> print(completeness_score([0, 1, 2, 3], [0, 0, 1, 1]))
      1.0

    If classes members are split across different clusters, the
    assignment cannot be complete::

      >>> print(completeness_score([0, 0, 1, 1], [0, 1, 0, 1]))
      0.0
      >>> print(completeness_score([0, 0, 0, 0], [0, 1, 2, 3]))
      0.0

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred)[1]

def v_measure_score(labels_true, labels_pred):
    """V-measure cluster labeling given a ground truth.

    This score is identical to :func:`normalized_mutual_info_score`.

    The V-measure is the harmonic mean between homogeneity and completeness::

        v = 2 * (homogeneity * completeness) / (homogeneity + completeness)

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <homogeneity_completeness>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        ground truth class labels to be used as a reference

    labels_pred : array, shape = [n_samples]
        cluster labels to evaluate

    Returns
    -------
    v_measure: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    References
    ----------

    .. [1] `Andrew Rosenberg and Julia Hirschberg, 2007. V-Measure: A
       conditional entropy-based external cluster evaluation measure
       <http://aclweb.org/anthology/D/D07/D07-1043.pdf>`_

    See also
    --------
    homogeneity_score
    completeness_score

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have score 1.0::

      >>> from sklearn.metrics.cluster import v_measure_score
      >>> v_measure_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> v_measure_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    Labelings that assign all classes members to the same clusters
    are complete be not homogeneous, hence penalized::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 2], [0, 0, 1, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.8...
      >>> print("%.6f" % v_measure_score([0, 1, 2, 3], [0, 0, 1, 1]))
      ...                                                  # doctest: +ELLIPSIS
      0.66...

    Labelings that have pure clusters with members coming from the same
    classes are homogeneous but un-necessary splits harms completeness
    and thus penalize V-measure as well::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 1, 2]))
      ...                                                  # doctest: +ELLIPSIS
      0.8...
      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      0.66...

    If classes members are completely split across different clusters,
    the assignment is totally incomplete, hence the V-Measure is null::

      >>> print("%.6f" % v_measure_score([0, 0, 0, 0], [0, 1, 2, 3]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    Clusters that include samples from totally different classes totally
    destroy the homogeneity of the labeling, hence::

      >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 0, 0]))
      ...                                                  # doctest: +ELLIPSIS
      0.0...

    """
    return homogeneity_completeness_v_measure(labels_true, labels_pred)[2]

def mutual_info_score(labels_true, labels_pred, contingency=None):
    """Mutual Information between two clusterings

    The Mutual Information is a measure of the similarity between two labels of
    the same data. Where :math:`P(i)` is the probability of a random sample
    occurring in cluster :math:`U_i` and :math:`P'(j)` is the probability of a
    random sample occurring in cluster :math:`V_j`, the Mutual Information
    between clusterings :math:`U` and :math:`V` is given as:

    .. math::

        MI(U,V)=\sum_{i=1}^R \sum_{j=1}^C P(i,j)\log\\frac{P(i,j)}{P(i)P'(j)}

    This is equal to the Kullback-Leibler divergence of the joint distribution
    with the product distribution of the marginals.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    contingency: None or array, shape = [n_classes_true, n_classes_pred]
        A contingency matrix given by the :func:`contingency_matrix` function.
        If value is ``None``, it will be computed, otherwise the given value is
        used, with ``labels_true`` and ``labels_pred`` ignored.

    Returns
    -------
    mi: float
       Mutual information, a non-negative value

    See also
    --------
    adjusted_mutual_info_score: Adjusted against chance Mutual Information
    normalized_mutual_info_score: Normalized Mutual Information
    """
    if contingency is None:
        labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
        contingency = contingency_matrix(labels_true, labels_pred)
    contingency = np.array(contingency, dtype='float')
    contingency_sum = np.sum(contingency)
    pi = np.sum(contingency, axis=1)
    pj = np.sum(contingency, axis=0)
    outer = np.outer(pi, pj)
    nnz = contingency != 0.0
    # normalized contingency
    contingency_nm = contingency[nnz]
    log_contingency_nm = np.log(contingency_nm)
    contingency_nm /= contingency_sum
    # log(a / b) should be calculated as log(a) - log(b) for
    # possible loss of precision
    log_outer = -np.log(outer[nnz]) + log(pi.sum()) + log(pj.sum())
    mi = (contingency_nm * (log_contingency_nm - log(contingency_sum))
          + contingency_nm * log_outer)
    return mi.sum()

def adjusted_mutual_info_score(labels_true, labels_pred):
    """Adjusted Mutual Information between two clusterings

    Adjusted Mutual Information (AMI) is an adjustment of the Mutual
    Information (MI) score to account for chance. It accounts for the fact that
    the MI is generally higher for two clusterings with a larger number of
    clusters, regardless of whether there is actually more information shared.
    For two clusterings :math:`U` and :math:`V`, the AMI is given as::

        AMI(U, V) = [MI(U, V) - E(MI(U, V))] / [max(H(U), H(V)) - E(MI(U, V))]

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Be mindful that this function is an order of magnitude slower than other
    metrics, such as the Adjusted Rand Index.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    Returns
    -------
    ami: float(upperlimited by 1.0)
       The AMI returns a value of 1 when the two partitions are identical
       (ie perfectly matched). Random partitions (independent labellings) have
       an expected AMI around 0 on average hence can be negative.

    See also
    --------
    adjusted_rand_score: Adjusted Rand Index
    mutual_information_score: Mutual Information (not adjusted for chance)

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have
    score 1.0::

      >>> from sklearn.metrics.cluster import adjusted_mutual_info_score
      >>> adjusted_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> adjusted_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    If classes members are completely split across different clusters,
    the assignment is totally in-complete, hence the AMI is null::

      >>> adjusted_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    References
    ----------
    .. [1] `Vinh, Epps, and Bailey, (2010). Information Theoretic Measures for
       Clusterings Comparison: Variants, Properties, Normalization and
       Correction for Chance, JMLR
       <http://jmlr.csail.mit.edu/papers/volume11/vinh10a/vinh10a.pdf>`_

    .. [2] `Wikipedia entry for the Adjusted Mutual Information
       <http://en.wikipedia.org/wiki/Adjusted_Mutual_Information>`_

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    n_samples = labels_true.shape[0]
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split.
    # This is a perfect match hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1
            or classes.shape[0] == clusters.shape[0] == 0):
        return 1.0
    contingency = contingency_matrix(labels_true, labels_pred)
    contingency = np.array(contingency, dtype='float')
    # Calculate the MI for the two clusterings
    mi = mutual_info_score(labels_true, labels_pred,
                           contingency=contingency)
    # Calculate the expected value for the mutual information
    emi = expected_mutual_information(contingency, n_samples)
    # Calculate entropy for each labeling
    h_true, h_pred = entropy(labels_true), entropy(labels_pred)
    ami = (mi - emi) / (max(h_true, h_pred) - emi)
    return ami

def normalized_mutual_info_score(labels_true, labels_pred):
    """Normalized Mutual Information between two clusterings

    Normalized Mutual Information (NMI) is an normalization of the Mutual
    Information (MI) score to scale the results between 0 (no mutual
    information) and 1 (perfect correlation). In this function, mutual
    information is normalized by ``sqrt(H(labels_true) * H(labels_pred))``

    This measure is not adjusted for chance. Therefore
    :func:`adjusted_mustual_info_score` might be preferred.

    This metric is independent of the absolute values of the labels:
    a permutation of the class or cluster label values won't change the
    score value in any way.

    This metric is furthermore symmetric: switching ``label_true`` with
    ``label_pred`` will return the same score value. This can be useful to
    measure the agreement of two independent label assignments strategies
    on the same dataset when the real ground truth is not known.

    Read more in the :ref:`User Guide <mutual_info_score>`.

    Parameters
    ----------
    labels_true : int array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    labels_pred : array, shape = [n_samples]
        A clustering of the data into disjoint subsets.

    Returns
    -------
    nmi: float
       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling

    See also
    --------
    adjusted_rand_score: Adjusted Rand Index
    adjusted_mutual_info_score: Adjusted Mutual Information (adjusted
        against chance)

    Examples
    --------

    Perfect labelings are both homogeneous and complete, hence have
    score 1.0::

      >>> from sklearn.metrics.cluster import normalized_mutual_info_score
      >>> normalized_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
      1.0
      >>> normalized_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
      1.0

    If classes members are completely split across different clusters,
    the assignment is totally in-complete, hence the NMI is null::

      >>> normalized_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
      0.0

    """
    labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
    classes = np.unique(labels_true)
    clusters = np.unique(labels_pred)
    # Special limit cases: no clustering since the data is not split.
    # This is a perfect match hence return 1.0.
    if (classes.shape[0] == clusters.shape[0] == 1
            or classes.shape[0] == clusters.shape[0] == 0):
        return 1.0
    contingency = contingency_matrix(labels_true, labels_pred)
    contingency = np.array(contingency, dtype='float')
    # Calculate the MI for the two clusterings
    mi = mutual_info_score(labels_true, labels_pred,
                           contingency=contingency)
    # Calculate the expected value for the mutual information
    # Calculate entropy for each labeling
    h_true, h_pred = entropy(labels_true), entropy(labels_pred)
    nmi = mi / max(np.sqrt(h_true * h_pred), 1e-10)
    return nmi

def entropy(labels):
    """Calculates the entropy for a labeling."""
    if len(labels) == 0:
        return 1.0
    label_idx = np.unique(labels, return_inverse=True)[1]
    pi = bincount(label_idx).astype(np.float)
    pi = pi[pi > 0]
    pi_sum = np.sum(pi)
    # log(a / b) should be calculated as log(a) - log(b) for
    # possible loss of precision
    return -np.sum((pi / pi_sum) * (np.log(pi) - log(pi_sum)))