"""Test the cross_validation module"""
from __future__ import division
import warnings

import numpy as np
from scipy.sparse import coo_matrix
from scipy.sparse import csr_matrix
from scipy import stats

from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_not_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.mocking import CheckingClassifier, MockDataFrame

with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    from sklearn import cross_validation as cval

from sklearn.datasets import make_regression
from sklearn.datasets import load_boston
from sklearn.datasets import load_digits
from sklearn.datasets import load_iris
from sklearn.datasets import make_multilabel_classification
from sklearn.metrics import explained_variance_score
from sklearn.metrics import make_scorer
from sklearn.metrics import precision_score
from sklearn.externals import six
from sklearn.externals.six.moves import zip

from sklearn.linear_model import Ridge
from sklearn.multiclass import OneVsRestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.cluster import KMeans

from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline

class MockClassifier(object):
    """Dummy classifier to test the cross-validation"""

    def __init__(self, a=0, allow_nd=False):
        self.a = a
        self.allow_nd = allow_nd

    def fit(self, X, Y=None, sample_weight=None, class_prior=None,
            sparse_sample_weight=None, sparse_param=None, dummy_int=None,
            dummy_str=None, dummy_obj=None, callback=None):
        """The dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        """
        self.dummy_int = dummy_int
        self.dummy_str = dummy_str
        self.dummy_obj = dummy_obj
        if callback is not None:
            callback(self)

        if self.allow_nd:
            X = X.reshape(len(X), -1)
        if X.ndim >= 3 and not self.allow_nd:
            raise ValueError('X cannot be d')
        if sample_weight is not None:
            assert_true(sample_weight.shape[0] == X.shape[0],
                        'MockClassifier extra fit_param sample_weight.shape[0]'
                        ' is {0}, should be {1}'.format(sample_weight.shape[0],
                                                        X.shape[0]))
        if class_prior is not None:
            assert_true(class_prior.shape[0] == len(np.unique(y)),
                        'MockClassifier extra fit_param class_prior.shape[0]'
                        ' is {0}, should be {1}'.format(class_prior.shape[0],
                                                        len(np.unique(y))))
        if sparse_sample_weight is not None:
            fmt = ('MockClassifier extra fit_param sparse_sample_weight'
                   '.shape[0] is {0}, should be {1}')
            assert_true(sparse_sample_weight.shape[0] == X.shape[0],
                        fmt.format(sparse_sample_weight.shape[0], X.shape[0]))
        if sparse_param is not None:
            fmt = ('MockClassifier extra fit_param sparse_param.shape '
                   'is ({0}, {1}), should be ({2}, {3})')
            assert_true(sparse_param.shape == P_sparse.shape,
                        fmt.format(sparse_param.shape[0],
                                   sparse_param.shape[1],
                                   P_sparse.shape[0], P_sparse.shape[1]))
        return self

    def predict(self, T):
        if self.allow_nd:
            T = T.reshape(len(T), -1)
        return T[:, 0]

    def score(self, X=None, Y=None):
        return 1. / (1 + np.abs(self.a))

    def get_params(self, deep=False):
        return {'a': self.a, 'allow_nd': self.allow_nd}

X = np.ones((10, 2))
X_sparse = coo_matrix(X)
W_sparse = coo_matrix((np.array([1]), (np.array([1]), np.array([0]))),
                      shape=(10, 1))
P_sparse = coo_matrix(np.eye(5))

# avoid StratifiedKFold's Warning about least populated class in y
y = np.arange(10) % 3

##############################################################################
# Tests

def check_valid_split(train, test, n_samples=None):
    # Use python sets to get more informative assertion failure messages
    train, test = set(train), set(test)

    # Train and test split should not overlap
    assert_equal(train.intersection(test), set())

    if n_samples is not None:
        # Check that the union of train an test split cover all the indices
        assert_equal(train.union(test), set(range(n_samples)))

def check_cv_coverage(cv, expected_n_iter=None, n_samples=None):
    # Check that a all the samples appear at least once in a test fold
    if expected_n_iter is not None:
        assert_equal(len(cv), expected_n_iter)
    else:
        expected_n_iter = len(cv)

    collected_test_samples = set()
    iterations = 0
    for train, test in cv:
        check_valid_split(train, test, n_samples=n_samples)
        iterations += 1
        collected_test_samples.update(test)

    # Check that the accumulated test samples cover the whole dataset
    assert_equal(iterations, expected_n_iter)
    if n_samples is not None:
        assert_equal(collected_test_samples, set(range(n_samples)))

def test_kfold_valueerrors():
    # Check that errors are raised if there is not enough samples
    assert_raises(ValueError, cval.KFold, 3, 4)

    # Check that a warning is raised if the least populated class has too few
    # members.
    y = [3, 3, -1, -1, 2]

    cv = assert_warns_message(Warning, "The least populated class",
                              cval.StratifiedKFold, y, 3)

    # Check that despite the warning the folds are still computed even
    # though all the classes are not necessarily represented at on each
    # side of the split at each split
    check_cv_coverage(cv, expected_n_iter=3, n_samples=len(y))

    # Error when number of folds is <= 1
    assert_raises(ValueError, cval.KFold, 2, 0)
    assert_raises(ValueError, cval.KFold, 2, 1)
    assert_raises(ValueError, cval.StratifiedKFold, y, 0)
    assert_raises(ValueError, cval.StratifiedKFold, y, 1)

    # When n is not integer:
    assert_raises(ValueError, cval.KFold, 2.5, 2)

    # When n_folds is not integer:
    assert_raises(ValueError, cval.KFold, 5, 1.5)
    assert_raises(ValueError, cval.StratifiedKFold, y, 1.5)

def test_kfold_indices():
    # Check all indices are returned in the test folds
    kf = cval.KFold(300, 3)
    check_cv_coverage(kf, expected_n_iter=3, n_samples=300)

    # Check all indices are returned in the test folds even when equal-sized
    # folds are not possible
    kf = cval.KFold(17, 3)
    check_cv_coverage(kf, expected_n_iter=3, n_samples=17)

def test_kfold_no_shuffle():
    # Manually check that KFold preserves the data ordering on toy datasets
    splits = iter(cval.KFold(4, 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 1])
    assert_array_equal(train, [2, 3])

    train, test = next(splits)
    assert_array_equal(test, [2, 3])
    assert_array_equal(train, [0, 1])

    splits = iter(cval.KFold(5, 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 1, 2])
    assert_array_equal(train, [3, 4])

    train, test = next(splits)
    assert_array_equal(test, [3, 4])
    assert_array_equal(train, [0, 1, 2])

def test_stratified_kfold_no_shuffle():
    # Manually check that StratifiedKFold preserves the data ordering as much
    # as possible on toy datasets in order to avoid hiding sample dependencies
    # when possible
    splits = iter(cval.StratifiedKFold([1, 1, 0, 0], 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 2])
    assert_array_equal(train, [1, 3])

    train, test = next(splits)
    assert_array_equal(test, [1, 3])
    assert_array_equal(train, [0, 2])

    splits = iter(cval.StratifiedKFold([1, 1, 1, 0, 0, 0, 0], 2))
    train, test = next(splits)
    assert_array_equal(test, [0, 1, 3, 4])
    assert_array_equal(train, [2, 5, 6])

    train, test = next(splits)
    assert_array_equal(test, [2, 5, 6])
    assert_array_equal(train, [0, 1, 3, 4])

def test_stratified_kfold_ratios():
    # Check that stratified kfold preserves label ratios in individual splits
    # Repeat with shuffling turned off and on
    n_samples = 1000
    labels = np.array([4] * int(0.10 * n_samples) +
                      [0] * int(0.89 * n_samples) +
                      [1] * int(0.01 * n_samples))
    for shuffle in [False, True]:
        for train, test in cval.StratifiedKFold(labels, 5, shuffle=shuffle):
            assert_almost_equal(np.sum(labels[train] == 4) / len(train), 0.10,
                                2)
            assert_almost_equal(np.sum(labels[train] == 0) / len(train), 0.89,
                                2)
            assert_almost_equal(np.sum(labels[train] == 1) / len(train), 0.01,
                                2)
            assert_almost_equal(np.sum(labels[test] == 4) / len(test), 0.10, 2)
            assert_almost_equal(np.sum(labels[test] == 0) / len(test), 0.89, 2)
            assert_almost_equal(np.sum(labels[test] == 1) / len(test), 0.01, 2)

def test_kfold_balance():
    # Check that KFold returns folds with balanced sizes
    for kf in [cval.KFold(i, 5) for i in range(11, 17)]:
        sizes = []
        for _, test in kf:
            sizes.append(len(test))

        assert_true((np.max(sizes) - np.min(sizes)) <= 1)
        assert_equal(np.sum(sizes), kf.n)

def test_stratifiedkfold_balance():
    # Check that KFold returns folds with balanced sizes (only when
    # stratification is possible)
    # Repeat with shuffling turned off and on
    labels = [0] * 3 + [1] * 14
    for shuffle in [False, True]:
        for skf in [cval.StratifiedKFold(labels[:i], 3, shuffle=shuffle)
                    for i in range(11, 17)]:
            sizes = []
            for _, test in skf:
                sizes.append(len(test))

            assert_true((np.max(sizes) - np.min(sizes)) <= 1)
            assert_equal(np.sum(sizes), skf.n)

def test_shuffle_kfold():
    # Check the indices are shuffled properly, and that all indices are
    # returned in the different test folds
    kf = cval.KFold(300, 3, shuffle=True, random_state=0)
    ind = np.arange(300)

    all_folds = None
    for train, test in kf:
        assert_true(np.any(np.arange(100) != ind[test]))
        assert_true(np.any(np.arange(100, 200) != ind[test]))
        assert_true(np.any(np.arange(200, 300) != ind[test]))

        if all_folds is None:
            all_folds = ind[test].copy()
        else:
            all_folds = np.concatenate((all_folds, ind[test]))

    all_folds.sort()
    assert_array_equal(all_folds, ind)

def test_shuffle_stratifiedkfold():
    # Check that shuffling is happening when requested, and for proper
    # sample coverage
    labels = [0] * 20 + [1] * 20
    kf0 = list(cval.StratifiedKFold(labels, 5, shuffle=True, random_state=0))
    kf1 = list(cval.StratifiedKFold(labels, 5, shuffle=True, random_state=1))
    for (_, test0), (_, test1) in zip(kf0, kf1):
        assert_true(set(test0) != set(test1))
    check_cv_coverage(kf0, expected_n_iter=5, n_samples=40)

def test_kfold_can_detect_dependent_samples_on_digits():  # see #2372
    # The digits samples are dependent: they are apparently grouped by authors
    # although we don't have any information on the groups segment locations
    # for this data. We can highlight this fact be computing k-fold cross-
    # validation with and without shuffling: we observe that the shuffling case
    # wrongly makes the IID assumption and is therefore too optimistic: it
    # estimates a much higher accuracy (around 0.96) than than the non
    # shuffling variant (around 0.86).

    digits = load_digits()
    X, y = digits.data[:800], digits.target[:800]
    model = SVC(C=10, gamma=0.005)
    n = len(y)

    cv = cval.KFold(n, 5, shuffle=False)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(0.88, mean_score)
    assert_greater(mean_score, 0.85)

    # Shuffling the data artificially breaks the dependency and hides the
    # overfitting of the model with regards to the writing style of the authors
    # by yielding a seriously overestimated score:

    cv = cval.KFold(n, 5, shuffle=True, random_state=0)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(mean_score, 0.95)

    cv = cval.KFold(n, 5, shuffle=True, random_state=1)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(mean_score, 0.95)

    # Similarly, StratifiedKFold should try to shuffle the data as little
    # as possible (while respecting the balanced class constraints)
    # and thus be able to detect the dependency by not overestimating
    # the CV score either. As the digits dataset is approximately balanced
    # the estimated mean score is close to the score measured with
    # non-shuffled KFold

    cv = cval.StratifiedKFold(y, 5)
    mean_score = cval.cross_val_score(model, X, y, cv=cv).mean()
    assert_greater(0.88, mean_score)
    assert_greater(mean_score, 0.85)

def test_label_kfold():
    rng = np.random.RandomState(0)

    # Parameters of the test
    n_labels = 15
    n_samples = 1000
    n_folds = 5

    # Construct the test data
    tolerance = 0.05 * n_samples  # 5 percent error allowed
    labels = rng.randint(0, n_labels, n_samples)
    folds = cval.LabelKFold(labels, n_folds=n_folds).idxs
    ideal_n_labels_per_fold = n_samples // n_folds

    # Check that folds have approximately the same size
    assert_equal(len(folds), len(labels))
    for i in np.unique(folds):
        assert_greater_equal(tolerance,
                             abs(sum(folds == i) - ideal_n_labels_per_fold))

    # Check that each label appears only in 1 fold
    for label in np.unique(labels):
        assert_equal(len(np.unique(folds[labels == label])), 1)

    # Check that no label is on both sides of the split
    labels = np.asarray(labels, dtype=object)
    for train, test in cval.LabelKFold(labels, n_folds=n_folds):
        assert_equal(len(np.intersect1d(labels[train], labels[test])), 0)

    # Construct the test data
    labels = ['Albert', 'Jean', 'Bertrand', 'Michel', 'Jean',
              'Francis', 'Robert', 'Michel', 'Rachel', 'Lois',
              'Michelle', 'Bernard', 'Marion', 'Laura', 'Jean',
              'Rachel', 'Franck', 'John', 'Gael', 'Anna', 'Alix',
              'Robert', 'Marion', 'David', 'Tony', 'Abel', 'Becky',
              'Madmood', 'Cary', 'Mary', 'Alexandre', 'David', 'Francis',
              'Barack', 'Abdoul', 'Rasha', 'Xi', 'Silvia']
    labels = np.asarray(labels, dtype=object)

    n_labels = len(np.unique(labels))
    n_samples = len(labels)
    n_folds = 5
    tolerance = 0.05 * n_samples  # 5 percent error allowed
    folds = cval.LabelKFold(labels, n_folds=n_folds).idxs
    ideal_n_labels_per_fold = n_samples // n_folds

    # Check that folds have approximately the same size
    assert_equal(len(folds), len(labels))
    for i in np.unique(folds):
        assert_greater_equal(tolerance,
                             abs(sum(folds == i) - ideal_n_labels_per_fold))

    # Check that each label appears only in 1 fold
    for label in np.unique(labels):
        assert_equal(len(np.unique(folds[labels == label])), 1)

    # Check that no label is on both sides of the split
    for train, test in cval.LabelKFold(labels, n_folds=n_folds):
        assert_equal(len(np.intersect1d(labels[train], labels[test])), 0)

    # Should fail if there are more folds than labels
    labels = np.array([1, 1, 1, 2, 2])
    assert_raises(ValueError, cval.LabelKFold, labels, n_folds=3)

def test_shuffle_split():
    ss1 = cval.ShuffleSplit(10, test_size=0.2, random_state=0)
    ss2 = cval.ShuffleSplit(10, test_size=2, random_state=0)
    ss3 = cval.ShuffleSplit(10, test_size=np.int32(2), random_state=0)
    for typ in six.integer_types:
        ss4 = cval.ShuffleSplit(10, test_size=typ(2), random_state=0)
    for t1, t2, t3, t4 in zip(ss1, ss2, ss3, ss4):
        assert_array_equal(t1[0], t2[0])
        assert_array_equal(t2[0], t3[0])
        assert_array_equal(t3[0], t4[0])
        assert_array_equal(t1[1], t2[1])
        assert_array_equal(t2[1], t3[1])
        assert_array_equal(t3[1], t4[1])

def test_stratified_shuffle_split_init():
    y = np.asarray([0, 1, 1, 1, 2, 2, 2])
    # Check that error is raised if there is a class with only one sample
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 0.2)

    # Check that error is raised if the test set size is smaller than n_classes
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 2)
    # Check that error is raised if the train set size is smaller than
    # n_classes
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 3, 2)

    y = np.asarray([0, 0, 0, 1, 1, 1, 2, 2, 2])
    # Check that errors are raised if there is not enough samples
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 0.5, 0.6)
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 8, 0.6)
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, 3, 0.6, 8)

    # Train size or test size too small
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, train_size=2)
    assert_raises(ValueError, cval.StratifiedShuffleSplit, y, test_size=2)

def test_stratified_shuffle_split_iter():
    ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
          np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
          np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
          np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
          np.array([-1] * 800 + [1] * 50)
          ]

    for y in ys:
        sss = cval.StratifiedShuffleSplit(y, 6, test_size=0.33,
                                          random_state=0)
        for train, test in sss:
            assert_array_equal(np.unique(y[train]), np.unique(y[test]))
            # Checks if folds keep classes proportions
            p_train = (np.bincount(np.unique(y[train], return_inverse=True)[1])
                       / float(len(y[train])))
            p_test = (np.bincount(np.unique(y[test], return_inverse=True)[1])
                      / float(len(y[test])))
            assert_array_almost_equal(p_train, p_test, 1)
            assert_equal(y[train].size + y[test].size, y.size)
            assert_array_equal(np.intersect1d(train, test), [])

def test_stratified_shuffle_split_even():
    # Test the StratifiedShuffleSplit, indices are drawn with a
    # equal chance
    n_folds = 5
    n_iter = 1000

    def assert_counts_are_ok(idx_counts, p):
        # Here we test that the distribution of the counts
        # per index is close enough to a binomial
        threshold = 0.05 / n_splits
        bf = stats.binom(n_splits, p)
        for count in idx_counts:
            p = bf.pmf(count)
            assert_true(p > threshold,
                        "An index is not drawn with chance corresponding "
                        "to even draws")

    for n_samples in (6, 22):
        labels = np.array((n_samples // 2) * [0, 1])
        splits = cval.StratifiedShuffleSplit(labels, n_iter=n_iter,
                                             test_size=1. / n_folds,
                                             random_state=0)

        train_counts = [0] * n_samples
        test_counts = [0] * n_samples
        n_splits = 0
        for train, test in splits:
            n_splits += 1
            for counter, ids in [(train_counts, train), (test_counts, test)]:
                for id in ids:
                    counter[id] += 1
        assert_equal(n_splits, n_iter)

        assert_equal(len(train), splits.n_train)
        assert_equal(len(test), splits.n_test)
        assert_equal(len(set(train).intersection(test)), 0)

        label_counts = np.unique(labels)
        assert_equal(splits.test_size, 1.0 / n_folds)
        assert_equal(splits.n_train + splits.n_test, len(labels))
        assert_equal(len(label_counts), 2)
        ex_test_p = float(splits.n_test) / n_samples
        ex_train_p = float(splits.n_train) / n_samples

        assert_counts_are_ok(train_counts, ex_train_p)
        assert_counts_are_ok(test_counts, ex_test_p)

def test_predefinedsplit_with_kfold_split():
    # Check that PredefinedSplit can reproduce a split generated by Kfold.
    folds = -1 * np.ones(10)
    kf_train = []
    kf_test = []
    for i, (train_ind, test_ind) in enumerate(cval.KFold(10, 5, shuffle=True)):
        kf_train.append(train_ind)
        kf_test.append(test_ind)
        folds[test_ind] = i
    ps_train = []
    ps_test = []
    ps = cval.PredefinedSplit(folds)
    for train_ind, test_ind in ps:
        ps_train.append(train_ind)
        ps_test.append(test_ind)
    assert_array_equal(ps_train, kf_train)
    assert_array_equal(ps_test, kf_test)

def test_label_shuffle_split():
    ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
          np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
          np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
          np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
          ]

    for y in ys:
        n_iter = 6
        test_size = 1. / 3
        slo = cval.LabelShuffleSplit(y, n_iter, test_size=test_size,
                                     random_state=0)

        # Make sure the repr works
        repr(slo)

        # Test that the length is correct
        assert_equal(len(slo), n_iter)

        y_unique = np.unique(y)

        for train, test in slo:
            # First test: no train label is in the test set and vice versa
            y_train_unique = np.unique(y[train])
            y_test_unique = np.unique(y[test])
            assert_false(np.any(np.in1d(y[train], y_test_unique)))
            assert_false(np.any(np.in1d(y[test], y_train_unique)))

            # Second test: train and test add up to all the data
            assert_equal(y[train].size + y[test].size, y.size)

            # Third test: train and test are disjoint
            assert_array_equal(np.intersect1d(train, test), [])

            # Fourth test: # unique train and test labels are correct,
            #              +- 1 for rounding error
            assert_true(abs(len(y_test_unique) -
                            round(test_size * len(y_unique))) <= 1)
            assert_true(abs(len(y_train_unique) -
                            round((1.0 - test_size) * len(y_unique))) <= 1)

def test_leave_label_out_changing_labels():
    # Check that LeaveOneLabelOut and LeavePLabelOut work normally if
    # the labels variable is changed before calling __iter__
    labels = np.array([0, 1, 2, 1, 1, 2, 0, 0])
    labels_changing = np.array(labels, copy=True)
    lolo = cval.LeaveOneLabelOut(labels)
    lolo_changing = cval.LeaveOneLabelOut(labels_changing)
    lplo = cval.LeavePLabelOut(labels, p=2)
    lplo_changing = cval.LeavePLabelOut(labels_changing, p=2)
    labels_changing[:] = 0
    for llo, llo_changing in [(lolo, lolo_changing), (lplo, lplo_changing)]:
        for (train, test), (train_chan, test_chan) in zip(llo, llo_changing):
            assert_array_equal(train, train_chan)
            assert_array_equal(test, test_chan)

def test_cross_val_score():
    clf = MockClassifier()
    for a in range(-10, 10):
        clf.a = a
        # Smoke test
        scores = cval.cross_val_score(clf, X, y)
        assert_array_equal(scores, clf.score(X, y))

        # test with multioutput y
        scores = cval.cross_val_score(clf, X_sparse, X)
        assert_array_equal(scores, clf.score(X_sparse, X))

        scores = cval.cross_val_score(clf, X_sparse, y)
        assert_array_equal(scores, clf.score(X_sparse, y))

        # test with multioutput y
        scores = cval.cross_val_score(clf, X_sparse, X)
        assert_array_equal(scores, clf.score(X_sparse, X))

    # test with X and y as list
    list_check = lambda x: isinstance(x, list)
    clf = CheckingClassifier(check_X=list_check)
    scores = cval.cross_val_score(clf, X.tolist(), y.tolist())

    clf = CheckingClassifier(check_y=list_check)
    scores = cval.cross_val_score(clf, X, y.tolist())

    assert_raises(ValueError, cval.cross_val_score, clf, X, y,
                  scoring="sklearn")

    # test with 3d X and
    X_3d = X[:, :, np.newaxis]
    clf = MockClassifier(allow_nd=True)
    scores = cval.cross_val_score(clf, X_3d, y)

    clf = MockClassifier(allow_nd=False)
    assert_raises(ValueError, cval.cross_val_score, clf, X_3d, y)

def test_cross_val_score_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((Series, DataFrame))
    except ImportError:
        pass
    for TargetType, InputFeatureType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)
        cval.cross_val_score(clf, X_df, y_ser)

def test_cross_val_score_mask():
    # test that cross_val_score works with boolean masks
    svm = SVC(kernel="linear")
    iris = load_iris()
    X, y = iris.data, iris.target
    cv_indices = cval.KFold(len(y), 5)
    scores_indices = cval.cross_val_score(svm, X, y, cv=cv_indices)
    cv_indices = cval.KFold(len(y), 5)
    cv_masks = []
    for train, test in cv_indices:
        mask_train = np.zeros(len(y), dtype=np.bool)
        mask_test = np.zeros(len(y), dtype=np.bool)
        mask_train[train] = 1
        mask_test[test] = 1
        cv_masks.append((train, test))
    scores_masks = cval.cross_val_score(svm, X, y, cv=cv_masks)
    assert_array_equal(scores_indices, scores_masks)

def test_cross_val_score_precomputed():
    # test for svm with precomputed kernel
    svm = SVC(kernel="precomputed")
    iris = load_iris()
    X, y = iris.data, iris.target
    linear_kernel = np.dot(X, X.T)
    score_precomputed = cval.cross_val_score(svm, linear_kernel, y)
    svm = SVC(kernel="linear")
    score_linear = cval.cross_val_score(svm, X, y)
    assert_array_equal(score_precomputed, score_linear)

    # Error raised for non-square X
    svm = SVC(kernel="precomputed")
    assert_raises(ValueError, cval.cross_val_score, svm, X, y)

    # test error is raised when the precomputed kernel is not array-like
    # or sparse
    assert_raises(ValueError, cval.cross_val_score, svm,
                  linear_kernel.tolist(), y)

def test_cross_val_score_fit_params():
    clf = MockClassifier()
    n_samples = X.shape[0]
    n_classes = len(np.unique(y))

    DUMMY_INT = 42
    DUMMY_STR = '42'
    DUMMY_OBJ = object()

    def assert_fit_params(clf):
        # Function to test that the values are passed correctly to the
        # classifier arguments for non-array type

        assert_equal(clf.dummy_int, DUMMY_INT)
        assert_equal(clf.dummy_str, DUMMY_STR)
        assert_equal(clf.dummy_obj, DUMMY_OBJ)

    fit_params = {'sample_weight': np.ones(n_samples),
                  'class_prior': np.ones(n_classes) / n_classes,
                  'sparse_sample_weight': W_sparse,
                  'sparse_param': P_sparse,
                  'dummy_int': DUMMY_INT,
                  'dummy_str': DUMMY_STR,
                  'dummy_obj': DUMMY_OBJ,
                  'callback': assert_fit_params}
    cval.cross_val_score(clf, X, y, fit_params=fit_params)

def test_cross_val_score_score_func():
    clf = MockClassifier()
    _score_func_args = []

    def score_func(y_test, y_predict):
        _score_func_args.append((y_test, y_predict))
        return 1.0

    with warnings.catch_warnings(record=True):
        scoring = make_scorer(score_func)
        score = cval.cross_val_score(clf, X, y, scoring=scoring)
    assert_array_equal(score, [1.0, 1.0, 1.0])
    assert len(_score_func_args) == 3

def test_cross_val_score_errors():
    class BrokenEstimator:
        pass

    assert_raises(TypeError, cval.cross_val_score, BrokenEstimator(), X)

def test_train_test_split_errors():
    assert_raises(ValueError, cval.train_test_split)
    assert_raises(ValueError, cval.train_test_split, range(3), train_size=1.1)
    assert_raises(ValueError, cval.train_test_split, range(3), test_size=0.6,
                  train_size=0.6)
    assert_raises(ValueError, cval.train_test_split, range(3),
                  test_size=np.float32(0.6), train_size=np.float32(0.6))
    assert_raises(ValueError, cval.train_test_split, range(3),
                  test_size="wrong_type")
    assert_raises(ValueError, cval.train_test_split, range(3), test_size=2,
                  train_size=4)
    assert_raises(TypeError, cval.train_test_split, range(3),
                  some_argument=1.1)
    assert_raises(ValueError, cval.train_test_split, range(3), range(42))

def test_train_test_split():
    X = np.arange(100).reshape((10, 10))
    X_s = coo_matrix(X)
    y = np.arange(10)

    # simple test
    split = cval.train_test_split(X, y, test_size=None, train_size=.5)
    X_train, X_test, y_train, y_test = split
    assert_equal(len(y_test), len(y_train))
    # test correspondence of X and y
    assert_array_equal(X_train[:, 0], y_train * 10)
    assert_array_equal(X_test[:, 0], y_test * 10)

    # conversion of lists to arrays (deprecated?)
    with warnings.catch_warnings(record=True):
        split = cval.train_test_split(X, X_s, y.tolist())
    X_train, X_test, X_s_train, X_s_test, y_train, y_test = split
    assert_array_equal(X_train, X_s_train.toarray())
    assert_array_equal(X_test, X_s_test.toarray())

    # don't convert lists to anything else by default
    split = cval.train_test_split(X, X_s, y.tolist())
    X_train, X_test, X_s_train, X_s_test, y_train, y_test = split
    assert_true(isinstance(y_train, list))
    assert_true(isinstance(y_test, list))

    # allow nd-arrays
    X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
    y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
    split = cval.train_test_split(X_4d, y_3d)
    assert_equal(split[0].shape, (7, 5, 3, 2))
    assert_equal(split[1].shape, (3, 5, 3, 2))
    assert_equal(split[2].shape, (7, 7, 11))
    assert_equal(split[3].shape, (3, 7, 11))

    # test stratification option
    y = np.array([1, 1, 1, 1, 2, 2, 2, 2])
    for test_size, exp_test_size in zip([2, 4, 0.25, 0.5, 0.75],
                                        [2, 4, 2, 4, 6]):
        train, test = cval.train_test_split(y,
                                            test_size=test_size,
                                            stratify=y,
                                            random_state=0)
        assert_equal(len(test), exp_test_size)
        assert_equal(len(test) + len(train), len(y))
        # check the 1:1 ratio of ones and twos in the data is preserved
        assert_equal(np.sum(train == 1), np.sum(train == 2))

def train_test_split_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [MockDataFrame]
    try:
        from pandas import DataFrame
        types.append(DataFrame)
    except ImportError:
        pass
    for InputFeatureType in types:
        # X dataframe
        X_df = InputFeatureType(X)
        X_train, X_test = cval.train_test_split(X_df)
        assert_true(isinstance(X_train, InputFeatureType))
        assert_true(isinstance(X_test, InputFeatureType))

def train_test_split_mock_pandas():
    # X mock dataframe
    X_df = MockDataFrame(X)
    X_train, X_test = cval.train_test_split(X_df)
    assert_true(isinstance(X_train, MockDataFrame))
    assert_true(isinstance(X_test, MockDataFrame))

def test_cross_val_score_with_score_func_classification():
    iris = load_iris()
    clf = SVC(kernel='linear')

    # Default score (should be the accuracy score)
    scores = cval.cross_val_score(clf, iris.data, iris.target, cv=5)
    assert_array_almost_equal(scores, [0.97, 1., 0.97, 0.97, 1.], 2)

    # Correct classification score (aka. zero / one score) - should be the
    # same as the default estimator score
    zo_scores = cval.cross_val_score(clf, iris.data, iris.target,
                                     scoring="accuracy", cv=5)
    assert_array_almost_equal(zo_scores, [0.97, 1., 0.97, 0.97, 1.], 2)

    # F1 score (class are balanced so f1_score should be equal to zero/one
    # score
    f1_scores = cval.cross_val_score(clf, iris.data, iris.target,
                                     scoring="f1_weighted", cv=5)
    assert_array_almost_equal(f1_scores, [0.97, 1., 0.97, 0.97, 1.], 2)

def test_cross_val_score_with_score_func_regression():
    X, y = make_regression(n_samples=30, n_features=20, n_informative=5,
                           random_state=0)
    reg = Ridge()

    # Default score of the Ridge regression estimator
    scores = cval.cross_val_score(reg, X, y, cv=5)
    assert_array_almost_equal(scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)

    # R2 score (aka. determination coefficient) - should be the
    # same as the default estimator score
    r2_scores = cval.cross_val_score(reg, X, y, scoring="r2", cv=5)
    assert_array_almost_equal(r2_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)

    # Mean squared error; this is a loss function, so "scores" are negative
    mse_scores = cval.cross_val_score(reg, X, y, cv=5,
                                      scoring="mean_squared_error")
    expected_mse = np.array([-763.07, -553.16, -274.38, -273.26, -1681.99])
    assert_array_almost_equal(mse_scores, expected_mse, 2)

    # Explained variance
    scoring = make_scorer(explained_variance_score)
    ev_scores = cval.cross_val_score(reg, X, y, cv=5, scoring=scoring)
    assert_array_almost_equal(ev_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)

def test_permutation_score():
    iris = load_iris()
    X = iris.data
    X_sparse = coo_matrix(X)
    y = iris.target
    svm = SVC(kernel='linear')
    cv = cval.StratifiedKFold(y, 2)

    score, scores, pvalue = cval.permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")
    assert_greater(score, 0.9)
    assert_almost_equal(pvalue, 0.0, 1)

    score_label, _, pvalue_label = cval.permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy",
        labels=np.ones(y.size), random_state=0)
    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # check that we obtain the same results with a sparse representation
    svm_sparse = SVC(kernel='linear')
    cv_sparse = cval.StratifiedKFold(y, 2)
    score_label, _, pvalue_label = cval.permutation_test_score(
        svm_sparse, X_sparse, y, n_permutations=30, cv=cv_sparse,
        scoring="accuracy", labels=np.ones(y.size), random_state=0)

    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # test with custom scoring object
    def custom_score(y_true, y_pred):
        return (((y_true == y_pred).sum() - (y_true != y_pred).sum())
                / y_true.shape[0])

    scorer = make_scorer(custom_score)
    score, _, pvalue = cval.permutation_test_score(
        svm, X, y, n_permutations=100, scoring=scorer, cv=cv, random_state=0)
    assert_almost_equal(score, .93, 2)
    assert_almost_equal(pvalue, 0.01, 3)

    # set random y
    y = np.mod(np.arange(len(y)), 3)

    score, scores, pvalue = cval.permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")

    assert_less(score, 0.5)
    assert_greater(pvalue, 0.2)

def test_cross_val_generator_with_indices():
    X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    y = np.array([1, 1, 2, 2])
    labels = np.array([1, 2, 3, 4])
    # explicitly passing indices value is deprecated
    loo = cval.LeaveOneOut(4)
    lpo = cval.LeavePOut(4, 2)
    kf = cval.KFold(4, 2)
    skf = cval.StratifiedKFold(y, 2)
    lolo = cval.LeaveOneLabelOut(labels)
    lopo = cval.LeavePLabelOut(labels, 2)
    ps = cval.PredefinedSplit([1, 1, 2, 2])
    ss = cval.ShuffleSplit(2)
    for cv in [loo, lpo, kf, skf, lolo, lopo, ss, ps]:
        for train, test in cv:
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            X[train], X[test]
            y[train], y[test]

@ignore_warnings
def test_cross_val_generator_with_default_indices():
    X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
    y = np.array([1, 1, 2, 2])
    labels = np.array([1, 2, 3, 4])
    loo = cval.LeaveOneOut(4)
    lpo = cval.LeavePOut(4, 2)
    kf = cval.KFold(4, 2)
    skf = cval.StratifiedKFold(y, 2)
    lolo = cval.LeaveOneLabelOut(labels)
    lopo = cval.LeavePLabelOut(labels, 2)
    ss = cval.ShuffleSplit(2)
    ps = cval.PredefinedSplit([1, 1, 2, 2])
    for cv in [loo, lpo, kf, skf, lolo, lopo, ss, ps]:
        for train, test in cv:
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            assert_not_equal(np.asarray(train).dtype.kind, 'b')
            X[train], X[test]
            y[train], y[test]

def test_shufflesplit_errors():
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=2.0)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=1.0)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=0.1,
                  train_size=0.95)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=11)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=10)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=8, train_size=3)
    assert_raises(ValueError, cval.ShuffleSplit, 10, train_size=1j)
    assert_raises(ValueError, cval.ShuffleSplit, 10, test_size=None,
                  train_size=None)

def test_shufflesplit_reproducible():
    # Check that iterating twice on the ShuffleSplit gives the same
    # sequence of train-test when the random_state is given
    ss = cval.ShuffleSplit(10, random_state=21)
    assert_array_equal(list(a for a, b in ss), list(a for a, b in ss))

def test_safe_split_with_precomputed_kernel():
    clf = SVC()
    clfp = SVC(kernel="precomputed")

    iris = load_iris()
    X, y = iris.data, iris.target
    K = np.dot(X, X.T)

    cv = cval.ShuffleSplit(X.shape[0], test_size=0.25, random_state=0)
    tr, te = list(cv)[0]

    X_tr, y_tr = cval._safe_split(clf, X, y, tr)
    K_tr, y_tr2 = cval._safe_split(clfp, K, y, tr)
    assert_array_almost_equal(K_tr, np.dot(X_tr, X_tr.T))

    X_te, y_te = cval._safe_split(clf, X, y, te, tr)
    K_te, y_te2 = cval._safe_split(clfp, K, y, te, tr)
    assert_array_almost_equal(K_te, np.dot(X_te, X_tr.T))

def test_cross_val_score_allow_nans():
    # Check that cross_val_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    cval.cross_val_score(p, X, y, cv=5)

def test_train_test_split_allow_nans():
    # Check that train_test_split allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    cval.train_test_split(X, y, test_size=0.2, random_state=42)

def test_permutation_test_score_allow_nans():
    # Check that permutation_test_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    cval.permutation_test_score(p, X, y, cv=5)

def test_check_cv_return_types():
    X = np.ones((9, 2))
    cv = cval.check_cv(3, X, classifier=False)
    assert_true(isinstance(cv, cval.KFold))

    y_binary = np.array([0, 1, 0, 1, 0, 0, 1, 1, 1])
    cv = cval.check_cv(3, X, y_binary, classifier=True)
    assert_true(isinstance(cv, cval.StratifiedKFold))

    y_multiclass = np.array([0, 1, 0, 1, 2, 1, 2, 0, 2])
    cv = cval.check_cv(3, X, y_multiclass, classifier=True)
    assert_true(isinstance(cv, cval.StratifiedKFold))

    X = np.ones((5, 2))
    y_multilabel = [[1, 0, 1], [1, 1, 0], [0, 0, 0], [0, 1, 1], [1, 0, 0]]
    cv = cval.check_cv(3, X, y_multilabel, classifier=True)
    assert_true(isinstance(cv, cval.KFold))

    y_multioutput = np.array([[1, 2], [0, 3], [0, 0], [3, 1], [2, 0]])
    cv = cval.check_cv(3, X, y_multioutput, classifier=True)
    assert_true(isinstance(cv, cval.KFold))

def test_cross_val_score_multilabel():
    X = np.array([[-3, 4], [2, 4], [3, 3], [0, 2], [-3, 1],
                  [-2, 1], [0, 0], [-2, -1], [-1, -2], [1, -2]])
    y = np.array([[1, 1], [0, 1], [0, 1], [0, 1], [1, 1],
                  [0, 1], [1, 0], [1, 1], [1, 0], [0, 0]])
    clf = KNeighborsClassifier(n_neighbors=1)
    scoring_micro = make_scorer(precision_score, average='micro')
    scoring_macro = make_scorer(precision_score, average='macro')
    scoring_samples = make_scorer(precision_score, average='samples')
    score_micro = cval.cross_val_score(clf, X, y, scoring=scoring_micro, cv=5)
    score_macro = cval.cross_val_score(clf, X, y, scoring=scoring_macro, cv=5)
    score_samples = cval.cross_val_score(clf, X, y,
                                         scoring=scoring_samples, cv=5)
    assert_almost_equal(score_micro, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 3])
    assert_almost_equal(score_macro, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 4])
    assert_almost_equal(score_samples, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 4])

def test_cross_val_predict():
    boston = load_boston()
    X, y = boston.data, boston.target
    cv = cval.KFold(len(boston.target))

    est = Ridge()

    # Naive loop (should be same as cross_val_predict):
    preds2 = np.zeros_like(y)
    for train, test in cv:
        est.fit(X[train], y[train])
        preds2[test] = est.predict(X[test])

    preds = cval.cross_val_predict(est, X, y, cv=cv)
    assert_array_almost_equal(preds, preds2)

    preds = cval.cross_val_predict(est, X, y)
    assert_equal(len(preds), len(y))

    cv = cval.LeaveOneOut(len(y))
    preds = cval.cross_val_predict(est, X, y, cv=cv)
    assert_equal(len(preds), len(y))

    Xsp = X.copy()
    Xsp *= (Xsp > np.median(Xsp))
    Xsp = coo_matrix(Xsp)
    preds = cval.cross_val_predict(est, Xsp, y)
    assert_array_almost_equal(len(preds), len(y))

    preds = cval.cross_val_predict(KMeans(), X)
    assert_equal(len(preds), len(y))

    def bad_cv():
        for i in range(4):
            yield np.array([0, 1, 2, 3]), np.array([4, 5, 6, 7, 8])

    assert_raises(ValueError, cval.cross_val_predict, est, X, y, cv=bad_cv())

def test_cross_val_predict_input_types():
    clf = Ridge()
    # Smoke test
    predictions = cval.cross_val_predict(clf, X, y)
    assert_equal(predictions.shape, (10,))

    # test with multioutput y
    predictions = cval.cross_val_predict(clf, X_sparse, X)
    assert_equal(predictions.shape, (10, 2))

    predictions = cval.cross_val_predict(clf, X_sparse, y)
    assert_array_equal(predictions.shape, (10,))

    # test with multioutput y
    predictions = cval.cross_val_predict(clf, X_sparse, X)
    assert_array_equal(predictions.shape, (10, 2))

    # test with X and y as list
    list_check = lambda x: isinstance(x, list)
    clf = CheckingClassifier(check_X=list_check)
    predictions = cval.cross_val_predict(clf, X.tolist(), y.tolist())

    clf = CheckingClassifier(check_y=list_check)
    predictions = cval.cross_val_predict(clf, X, y.tolist())

    # test with 3d X and
    X_3d = X[:, :, np.newaxis]
    check_3d = lambda x: x.ndim == 3
    clf = CheckingClassifier(check_X=check_3d)
    predictions = cval.cross_val_predict(clf, X_3d, y)
    assert_array_equal(predictions.shape, (10,))

def test_cross_val_predict_pandas():
    # check cross_val_score doesn't destroy pandas dataframe
    types = [(MockDataFrame, MockDataFrame)]
    try:
        from pandas import Series, DataFrame
        types.append((Series, DataFrame))
    except ImportError:
        pass
    for TargetType, InputFeatureType in types:
        # X dataframe, y series
        X_df, y_ser = InputFeatureType(X), TargetType(y)
        check_df = lambda x: isinstance(x, InputFeatureType)
        check_series = lambda x: isinstance(x, TargetType)
        clf = CheckingClassifier(check_X=check_df, check_y=check_series)
        cval.cross_val_predict(clf, X_df, y_ser)

def test_sparse_fit_params():
    iris = load_iris()
    X, y = iris.data, iris.target
    clf = MockClassifier()
    fit_params = {'sparse_sample_weight': coo_matrix(np.eye(X.shape[0]))}
    a = cval.cross_val_score(clf, X, y, fit_params=fit_params)
    assert_array_equal(a, np.ones(3))

def test_check_is_partition():
    p = np.arange(100)
    assert_true(cval._check_is_partition(p, 100))
    assert_false(cval._check_is_partition(np.delete(p, 23), 100))

    p[0] = 23
    assert_false(cval._check_is_partition(p, 100))

def test_cross_val_predict_sparse_prediction():
    # check that cross_val_predict gives same result for sparse and dense input
    X, y = make_multilabel_classification(n_classes=2, n_labels=1,
                                          allow_unlabeled=False,
                                          return_indicator=True,
                                          random_state=1)
    X_sparse = csr_matrix(X)
    y_sparse = csr_matrix(y)
    classif = OneVsRestClassifier(SVC(kernel='linear'))
    preds = cval.cross_val_predict(classif, X, y, cv=10)
    preds_sparse = cval.cross_val_predict(classif, X_sparse, y_sparse, cv=10)
    preds_sparse = preds_sparse.toarray()
    assert_array_almost_equal(preds_sparse, preds)


from .pls_ import _PLS

__all__ = ['CCA']

class CCA(_PLS):
    """CCA Canonical Correlation Analysis.

    CCA inherits from PLS with mode="B" and deflation_mode="canonical".

    Read more in the :ref:`User Guide <cross_decomposition>`.

    Parameters
    ----------
    n_components : int, (default 2).
        number of components to keep.

    scale : boolean, (default True)
        whether to scale the data?

    max_iter : an integer, (default 500)
        the maximum number of iterations of the NIPALS inner loop

    tol : non-negative real, default 1e-06.
        the tolerance used in the iterative algorithm

    copy : boolean
        Whether the deflation be done on a copy. Let the default value
        to True unless you don't care about side effects

    Attributes
    ----------
    x_weights_ : array, [p, n_components]
        X block weights vectors.

    y_weights_ : array, [q, n_components]
        Y block weights vectors.

    x_loadings_ : array, [p, n_components]
        X block loadings vectors.

    y_loadings_ : array, [q, n_components]
        Y block loadings vectors.

    x_scores_ : array, [n_samples, n_components]
        X scores.

    y_scores_ : array, [n_samples, n_components]
        Y scores.

    x_rotations_ : array, [p, n_components]
        X block to latents rotations.

    y_rotations_ : array, [q, n_components]
        Y block to latents rotations.

    n_iter_ : array-like
        Number of iterations of the NIPALS inner loop for each
        component.

    Notes
    -----
    For each component k, find the weights u, v that maximizes
    max corr(Xk u, Yk v), such that ``|u| = |v| = 1``

    Note that it maximizes only the correlations between the scores.

    The residual matrix of X (Xk+1) block is obtained by the deflation on the
    current X score: x_score.

    The residual matrix of Y (Yk+1) block is obtained by deflation on the
    current Y score.

    Examples
    --------
    >>> from sklearn.cross_decomposition import CCA
    >>> X = [[0., 0., 1.], [1.,0.,0.], [2.,2.,2.], [3.,5.,4.]]
    >>> Y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]]
    >>> cca = CCA(n_components=1)
    >>> cca.fit(X, Y)
    ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
    CCA(copy=True, max_iter=500, n_components=1, scale=True, tol=1e-06)
    >>> X_c, Y_c = cca.transform(X, Y)

    References
    ----------

    Jacob A. Wegelin. A survey of Partial Least Squares (PLS) methods, with
    emphasis on the two-block case. Technical Report 371, Department of
    Statistics, University of Washington, Seattle, 2000.

    In french but still a reference:
    Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris:
    Editions Technic.

    See also
    --------
    PLSCanonical
    PLSSVD
    """

    def __init__(self, n_components=2, scale=True,
                 max_iter=500, tol=1e-06, copy=True):
        _PLS.__init__(self, n_components=n_components, scale=scale,
                      deflation_mode="canonical", mode="B",
                      norm_y_weights=True, algorithm="nipals",
                      max_iter=max_iter, tol=tol, copy=copy)


# -*- coding: utf-8 -*-
import os

from encodings.aliases import aliases as _encodings

import pandas

from pandasqt.compat import Qt, QtCore, QtGui, Slot, Signal
from pandasqt.encoding import Detector
from pandasqt.models.DataFrameModel import DataFrameModel
from pandasqt.views.CustomDelegates import DtypeComboDelegate
from pandasqt.views._ui import icons_rc

from pandasqt.utils import fillNoneValues, convertTimestamps

class DelimiterValidator(QtGui.QRegExpValidator):
    """A Custom RegEx Validator.

    The validator checks, if the input has a length of 1.
    The input may contain any non-whitespace-character
    as denoted by the RegEx term `\S`.

    """

    def __init__(self, parent=None):
        """Constructs the object with the given parent.

        Args:
            parent (QObject, optional): Causes the objected to be owned
                by `parent` instead of Qt. Defaults to `None`.

        """
        super(DelimiterValidator, self).__init__(parent)
        re = QtCore.QRegExp('\S{1}')
        self.setRegExp(re)

class DelimiterSelectionWidget(QtGui.QGroupBox):
    """A custom widget with different text delimiter signs.

    A user can choose between 3 predefined and one user defined
    text delimiter characters. Default delimiters include `semicolon`,
    `colon` and `tabulator`. The user defined delimiter may only have
    a length of 1 and may not include any whitespace character.

    Attributes:
        delimiter (QtCore.pyqtSignal): This signal is emitted, whenever a
            delimiter character is selected by the user.
        semicolonRadioButton (QtGui.QRadioButton): A radio button to
            select the `semicolon` character as delimiter.
        commaRadioButton (QtGui.QRadioButton): A radio button to select
            the `comma` character as delimiter.
        tabRadioButton (QtGui.QRadioButton): A radio button to select
            the `tabulator` character as delimiter.
        otherRadioButton (QtGui.QRadioButton): A radio button to select
            the given input text as delimiter.
        otherSeparatorLineEdit (QtGui.QLineEdit): An input line to let the
            user enter one character only, which may be used as delimiter.

    """

    delimiter = Signal('QString')

    def __init__(self, parent=None):
        """Constructs the object with the given parent.

        Args:
            parent (QObject, optional): Causes the objected to be owned
                by `parent` instead of Qt. Defaults to `None`.

        """
        super(DelimiterSelectionWidget, self).__init__(parent)
        self.semicolonRadioButton = None
        self.commaRadioButton = None
        self.tabRadioButton = None
        self.otherRadioButton = None
        self.otherSeparatorLineEdit = None
        self._initUI()

    def _initUI(self):
        """Creates the inital layout with all subwidgets.

        The layout is a `QHBoxLayout`. Each time a radio button is
        selected or unselected, a slot
        `DelimiterSelectionWidget._delimiter` is called.
        Furthermore the `QLineEdit` widget has a custom regex validator
        `DelimiterValidator` enabled.

        """
        #layout = QtGui.QHBoxLayout(self)

        self.semicolonRadioButton = QtGui.QRadioButton(u'Semicolon')
        self.commaRadioButton = QtGui.QRadioButton(u'Comma')
        self.tabRadioButton = QtGui.QRadioButton(u'Tab')
        self.otherRadioButton = QtGui.QRadioButton(u'Other')
        self.semicolonRadioButton.setChecked(True)

        self.otherSeparatorLineEdit = QtGui.QLineEdit(self)
        self.otherSeparatorLineEdit.setEnabled(False)

        self.semicolonRadioButton.toggled.connect(self._delimiter)
        self.commaRadioButton.toggled.connect(self._delimiter)
        self.tabRadioButton.toggled.connect(self._delimiter)

        self.otherRadioButton.toggled.connect(self._enableLine)
        self.otherSeparatorLineEdit.textChanged.connect(lambda: self._delimiter(True))
        self.otherSeparatorLineEdit.setValidator(DelimiterValidator(self))

        currentLayout = self.layout()
        # unset and delete the current layout in order to set a new one
        if currentLayout is not None:
            del currentLayout

        layout = QtGui.QHBoxLayout()
        layout.addWidget(self.semicolonRadioButton)
        layout.addWidget(self.commaRadioButton)
        layout.addWidget(self.tabRadioButton)
        layout.addWidget(self.otherRadioButton)
        layout.addWidget(self.otherSeparatorLineEdit)
        self.setLayout(layout)

    @Slot('QBool')
    def _enableLine(self, toggled):
        self.otherSeparatorLineEdit.setEnabled(toggled)

    def currentSelected(self):
        """Returns the currently selected delimiter character.

        Returns:
            str: One of `,`, `;`, `\t`, `*other*`.

        """
        if self.commaRadioButton.isChecked():
            return ','
        elif self.semicolonRadioButton.isChecked():
            return ';'
        elif self.tabRadioButton.isChecked():
            return '\t'
        elif self.otherRadioButton.isChecked():
            return self.otherSeparatorLineEdit.text()
        return

    @Slot('QBool')
    def _delimiter(self, checked):
        if checked:
            if self.commaRadioButton.isChecked():
                self.delimiter.emit(',')
            elif self.semicolonRadioButton.isChecked():
                self.delimiter.emit(';')
            elif self.tabRadioButton.isChecked():
                self.delimiter.emit('\t')
            elif self.otherRadioButton.isChecked():
                ret = self.otherSeparatorLineEdit.text()
                if len(ret) > 0:
                    self.delimiter.emit(ret)

    def reset(self):
        """Resets this widget to its initial state.

        """
        self.semicolonRadioButton.setChecked(True)
        self.otherSeparatorLineEdit.setText('')

class CSVImportDialog(QtGui.QDialog):
    """A dialog to import any csv file into a pandas data frame.

    This modal dialog enables the user to enter any path to a csv
    file and parse this file with or without a header and with special
    delimiter characters.

    On a successful load, the data can be previewed and the column data
    types may be edited by the user.

    After all configuration is done, the dataframe and the underlying model
    may be used by the main application.

    Attributes:
        load (QtCore.pyqtSignal): This signal is emitted, whenever the
            dialog is successfully closed, e.g. when the ok button is
            pressed. Returns DataFrameModel and path of chosen csv file.
    """

    load = Signal('QAbstractItemModel', str)

    def __init__(self, parent=None):
        """Constructs the object with the given parent.

        Args:
            parent (QObject, optional): Causes the objected to be owned
                by `parent` instead of Qt. Defaults to `None`.

        """
        super(CSVImportDialog, self).__init__(parent)
        self._modal = True
        self._windowTitle = u'Import CSV'
        self._encodingKey = None
        self._filename = None
        self._delimiter = None
        self._header = None
        self._detector = Detector()
        self._initUI()

    def _initUI(self):
        """Initiates the user interface with a grid layout and several widgets.

        """
        self.setModal(self._modal)
        self.setWindowTitle(self._windowTitle)

        layout = QtGui.QGridLayout()

        self._filenameLabel = QtGui.QLabel(u'Choose File', self)
        self._filenameLineEdit = QtGui.QLineEdit(self)
        self._filenameLineEdit.textEdited.connect(self._updateFilename)
        chooseFileButtonIcon = QtGui.QIcon(QtGui.QPixmap(':/icons/document-open.png'))
        self._chooseFileAction = QtGui.QAction(self)
        self._chooseFileAction.setIcon(chooseFileButtonIcon)
        self._chooseFileAction.triggered.connect(self._openFile)

        self._chooseFileButton = QtGui.QToolButton(self)
        self._chooseFileButton.setDefaultAction(self._chooseFileAction)

        layout.addWidget(self._filenameLabel, 0, 0)
        layout.addWidget(self._filenameLineEdit, 0, 1, 1, 2)
        layout.addWidget(self._chooseFileButton, 0, 3)

        self._encodingLabel = QtGui.QLabel(u'File Encoding', self)

        encoding_names = map(lambda x: x.upper(), sorted(list(set(_encodings.viewvalues()))))
        self._encodingComboBox = QtGui.QComboBox(self)
        self._encodingComboBox.addItems(encoding_names)
        self._encodingComboBox.activated.connect(self._updateEncoding)

        layout.addWidget(self._encodingLabel, 1, 0)
        layout.addWidget(self._encodingComboBox, 1, 1, 1, 1)

        self._hasHeaderLabel = QtGui.QLabel(u'Header Available?', self)
        self._headerCheckBox = QtGui.QCheckBox(self)
        self._headerCheckBox.toggled.connect(self._updateHeader)

        layout.addWidget(self._hasHeaderLabel, 2, 0)
        layout.addWidget(self._headerCheckBox, 2, 1)

        self._delimiterLabel = QtGui.QLabel(u'Column Delimiter', self)
        self._delimiterBox = DelimiterSelectionWidget(self)
        self._delimiter = self._delimiterBox.currentSelected()
        self._delimiterBox.delimiter.connect(self._updateDelimiter)

        layout.addWidget(self._delimiterLabel, 3, 0)
        layout.addWidget(self._delimiterBox, 3, 1, 1, 3)

        self._tabWidget = QtGui.QTabWidget(self)
        self._previewTableView = QtGui.QTableView(self)
        self._datatypeTableView = QtGui.QTableView(self)
        self._tabWidget.addTab(self._previewTableView, u'Preview')
        self._tabWidget.addTab(self._datatypeTableView, u'Change Column Types')
        layout.addWidget(self._tabWidget, 4, 0, 3, 4)

        self._datatypeTableView.horizontalHeader().setDefaultSectionSize(200)
        self._datatypeTableView.setItemDelegateForColumn(1, DtypeComboDelegate(self._datatypeTableView))

        self._loadButton = QtGui.QPushButton(u'Load Data', self)
        #self.loadButton.setAutoDefault(False)

        self._cancelButton = QtGui.QPushButton(u'Cancel', self)
        # self.cancelButton.setDefault(False)
        # self.cancelButton.setAutoDefault(True)

        self._buttonBox = QtGui.QDialogButtonBox(self)
        self._buttonBox.addButton(self._loadButton, QtGui.QDialogButtonBox.AcceptRole)
        self._buttonBox.addButton(self._cancelButton, QtGui.QDialogButtonBox.RejectRole)
        self._buttonBox.accepted.connect(self.accepted)
        self._buttonBox.rejected.connect(self.rejected)
        layout.addWidget(self._buttonBox, 9, 2, 1, 2)
        self._loadButton.setDefault(False)
        self._filenameLineEdit.setFocus()

        self._statusBar = QtGui.QStatusBar(self)
        self._statusBar.setSizeGripEnabled(False)
        layout.addWidget(self._statusBar, 8, 0, 1, 4)
        self.setLayout(layout)

    @Slot('QString')
    def updateStatusBar(self, message):
        """Updates the status bar widget of this dialog with the given message.

        This method is also a `SLOT()`.
        The message will be shown for only 5 seconds.

        Args:
            message (QString): The new message which will be displayed.

        """
        self._statusBar.showMessage(message, 5000)

    @Slot()
    def _openFile(self):
        """Opens a file dialog and sets a value for the QLineEdit widget.

        This method is also a `SLOT`.

        """
        ret = QtGui.QFileDialog.getOpenFileName(self, self.tr(u'open file'), filter='Comma Separated Values (*.csv)')
        if ret:
            self._filenameLineEdit.setText(ret)
            self._updateFilename()

    @Slot('QBool')
    def _updateHeader(self, toggled):
        """Changes the internal flag, whether the csv file contains a header or not.

        This method is also a `SLOT`.

        In addition, after toggling the corresponding checkbox, the
        `_previewFile` method will be called.

        Args:
            toggled (boolean): A flag indicating the status of the checkbox.
                The flag will be used to update an internal variable.

        """
        self._header = 0 if toggled else None
        self._previewFile()

    @Slot()
    def _updateFilename(self):
        """Calls several methods after the filename changed.

        This method is also a `SLOT`.
        It checks the encoding of the changed filename and generates a
        preview of the data.

        """
        self._filename = self._filenameLineEdit.text()
        self._guessEncoding(self._filename)
        self._previewFile()

    def _guessEncoding(self, path):
        """Opens a file from the given `path` and checks the file encoding.

        The file must exists on the file system and end with the extension
        `.csv`. The file is read line by line until the encoding could be
        guessed.
        On a successfull identification, the widgets of this dialog will be
        updated.

        Args:
            path (string): Path to a csv file on the file system.

        """
        if os.path.exists(path) and path.lower().endswith('csv'):
            encoding = self._detector.detect(path)

            if encoding is not None:
                if encoding.startswith('utf'):
                    encoding = encoding.replace('-', '')
                encoding = encoding.replace('-','_')

                viewValue = _encodings.get(encoding)

                self._encodingKey = encoding

                index = self._encodingComboBox.findText(viewValue.upper())
                self._encodingComboBox.setCurrentIndex(index)

    @Slot('int')
    def _updateEncoding(self, index):
        """Changes the value of the encoding combo box to the value of given index.

        This method is also a `SLOT`.
        After the encoding is changed, the file will be reloaded and previewed.

        Args:
            index (int): An valid index of the combo box.

        """
        encoding = self._encodingComboBox.itemText(index)
        encoding = encoding.lower()

        self._encodingKey = _calculateEncodingKey(encoding)
        self._previewFile()

    @Slot('QString')
    def _updateDelimiter(self, delimiter):
        """Changes the value of the delimiter for the csv file.

        This method is also a `SLOT`.

        Args:
            delimiter (string): The new delimiter.

        """
        self._delimiter = delimiter
        self._previewFile()

    def _previewFile(self):
        """Updates the preview widgets with new models for both tab panes.

        """
        dataFrame = self._loadCSVDataFrame()
        dataFrameModel = DataFrameModel(dataFrame)
        dataFrameModel.enableEditing(True)
        self._previewTableView.setModel(dataFrameModel)
        columnModel = dataFrameModel.columnDtypeModel()
        columnModel.changeFailed.connect(self.updateStatusBar)
        self._datatypeTableView.setModel(columnModel)

    def _loadCSVDataFrame(self):
        """Loads the given csv file with pandas and generate a new dataframe.

        The file will be loaded with the configured encoding, delimiter
        and header.git
        If any execptions will occur, an empty Dataframe is generated
        and a message will appear in the status bar.

        Returns:
            pandas.DataFrame: A dataframe containing all the available
                information of the csv file.

        """
        if self._filename and os.path.exists(self._filename) and self._filename.endswith('.csv'):
            # default fallback if no encoding was found/selected
            encoding = self._encodingKey or 'uft8'

            try:
                dataFrame = pandas.read_csv(self._filename,
                    sep=self._delimiter, encoding=encoding,
                    header=self._header)
                dataFrame = dataFrame.apply(fillNoneValues)
                dataFrame = dataFrame.apply(convertTimestamps)
            except Exception, err:
                self.updateStatusBar(str(err))
                return pandas.DataFrame()
            self.updateStatusBar('Preview generated.')
            return dataFrame
        self.updateStatusBar('File does not exists or does not end with .csv')
        return pandas.DataFrame()

    def _resetWidgets(self):
        """Resets all widgets of this dialog to its inital state.

        """
        self._filenameLineEdit.setText('')
        self._encodingComboBox.setCurrentIndex(0)
        self._delimiterBox.reset()
        self._headerCheckBox.setChecked(False)
        self._statusBar.showMessage('')
        self._previewTableView.setModel(None)
        self._datatypeTableView.setModel(None)

    @Slot()
    def accepted(self):
        """Successfully close the widget and return the loaded model.

        This method is also a `SLOT`.
        The dialog will be closed, when the `ok` button is pressed. If
        a `DataFrame` was loaded, it will be emitted by the signal `load`.

        """
        model = self._previewTableView.model()
        if model is not None:
            df = model.dataFrame().copy()
            dfModel = DataFrameModel(df)
            self.load.emit(dfModel, self._filename)
        self._resetWidgets()
        self.accept()

    @Slot()
    def rejected(self):
        """Close the widget and reset its inital state.

        This method is also a `SLOT`.
        The dialog will be closed and all changes reverted, when the
        `cancel` button is pressed.

        """
        self._resetWidgets()
        self.reject()

class CSVExportDialog(QtGui.QDialog):
    """An widget to serialize a `DataFrameModel` to a `CSV-File`.

    """
    exported = Signal('QBool')

    def __init__(self, model=None, parent=None):
        super(CSVExportDialog, self).__init__(parent)
        self._model = model
        self._modal = True
        self._windowTitle = u'Export to CSV'
        self._idx = -1
        self._initUI()

    def _initUI(self):
        """Initiates the user interface with a grid layout and several widgets.

        """
        self.setModal(self._modal)
        self.setWindowTitle(self._windowTitle)

        layout = QtGui.QGridLayout()

        self._filenameLabel = QtGui.QLabel(u'Output File', self)
        self._filenameLineEdit = QtGui.QLineEdit(self)
        chooseFileButtonIcon = QtGui.QIcon(QtGui.QPixmap(':/icons/document-save-as.png'))
        self._chooseFileAction = QtGui.QAction(self)
        self._chooseFileAction.setIcon(chooseFileButtonIcon)
        self._chooseFileAction.triggered.connect(self._createFile)

        self._chooseFileButton = QtGui.QToolButton(self)
        self._chooseFileButton.setDefaultAction(self._chooseFileAction)

        layout.addWidget(self._filenameLabel, 0, 0)
        layout.addWidget(self._filenameLineEdit, 0, 1, 1, 2)
        layout.addWidget(self._chooseFileButton, 0, 3)

        self._encodingLabel = QtGui.QLabel(u'File Encoding', self)

        encoding_names = map(lambda x: x.upper(), sorted(list(set(_encodings.viewvalues()))))
        self._encodingComboBox = QtGui.QComboBox(self)
        self._encodingComboBox.addItems(encoding_names)
        self._idx = encoding_names.index('UTF_8')
        self._encodingComboBox.setCurrentIndex(self._idx)
        #self._encodingComboBox.activated.connect(self._updateEncoding)

        layout.addWidget(self._encodingLabel, 1, 0)
        layout.addWidget(self._encodingComboBox, 1, 1, 1, 1)

        self._hasHeaderLabel = QtGui.QLabel(u'Header Available?', self)
        self._headerCheckBox = QtGui.QCheckBox(self)
        #self._headerCheckBox.toggled.connect(self._updateHeader)

        layout.addWidget(self._hasHeaderLabel, 2, 0)
        layout.addWidget(self._headerCheckBox, 2, 1)

        self._delimiterLabel = QtGui.QLabel(u'Column Delimiter', self)
        self._delimiterBox = DelimiterSelectionWidget(self)

        layout.addWidget(self._delimiterLabel, 3, 0)
        layout.addWidget(self._delimiterBox, 3, 1, 1, 3)

        self._exportButton = QtGui.QPushButton(u'Export Data', self)
        self._cancelButton = QtGui.QPushButton(u'Cancel', self)

        self._buttonBox = QtGui.QDialogButtonBox(self)
        self._buttonBox.addButton(self._exportButton, QtGui.QDialogButtonBox.AcceptRole)
        self._buttonBox.addButton(self._cancelButton, QtGui.QDialogButtonBox.RejectRole)

        self._buttonBox.accepted.connect(self.accepted)
        self._buttonBox.rejected.connect(self.rejected)

        layout.addWidget(self._buttonBox, 5, 2, 1, 2)
        self._exportButton.setDefault(False)
        self._filenameLineEdit.setFocus()

        self._statusBar = QtGui.QStatusBar(self)
        self._statusBar.setSizeGripEnabled(False)
        layout.addWidget(self._statusBar, 4, 0, 1, 4)
        self.setLayout(layout)

    def setExportModel(self, model):
        if not isinstance(model, DataFrameModel):
            return False

        self._model = model
        return True

    @Slot()
    def _createFile(self):
        ret = QtGui.QFileDialog.getSaveFileName(self, 'Save File', filter='Comma Separated Value (*.csv)')
        self._filenameLineEdit.setText(ret)

    def _saveModel(self):
        delimiter = self._delimiterBox.currentSelected()
        header = self._headerCheckBox.isChecked() # column labels
        filename = self._filenameLineEdit.text()
        index = False # row labels

        encodingIndex = self._encodingComboBox.currentIndex()
        encoding = self._encodingComboBox.itemText(encodingIndex)
        encoding = _calculateEncodingKey(encoding.lower())

        try:
            dataFrame = self._model.dataFrame()
        except AttributeError, err:
            raise AttributeError('No data loaded to export.')
        else:
            try:
                dataFrame.to_csv(filename, encoding=encoding, header=header, index=index, sep=delimiter)
            except IOError, err:
                raise IOError('No filename given')
            except UnicodeError, err:
                raise UnicodeError('Could not encode all data. Choose a different encoding')
            except Exception:
                raise

    def _resetWidgets(self):
        """Resets all widgets of this dialog to its inital state.

        """
        self._filenameLineEdit.setText('')
        self._encodingComboBox.setCurrentIndex(self._idx)
        self._delimiterBox.reset()
        self._headerCheckBox.setChecked(False)
        self._statusBar.showMessage('')

    @Slot()
    def accepted(self):
        """Successfully close the widget and emit an export signal.

        This method is also a `SLOT`.
        The dialog will be closed, when the `Export Data` button is
        pressed. If errors occur during the export, the status bar
        will show the error message and the dialog will not be closed.

        """
        try:
            self._saveModel()
        except Exception, err:
            self._statusBar.showMessage(str(err))
        else:
            self._resetWidgets()
            self.exported.emit(True)
            self.accept()

    @Slot()
    def rejected(self):
        """Close the widget and reset its inital state.

        This method is also a `SLOT`.
        The dialog will be closed and all changes reverted, when the
        `cancel` button is pressed.

        """
        self._resetWidgets()
        self.exported.emit(False)
        self.reject()

def _calculateEncodingKey(comparator):
    """Gets the first key of all available encodings where the corresponding
    value matches the comparator.

    Args:
        comparator (string): A view name for an encoding.

    Returns:
        str: A key for a specific encoding used by python.

    """
    encodingName = None
    for k, v in _encodings.viewitems():
        if v == comparator:
            encodingName = k
            break
    return encodingName


'''
Computing the misclassification error distance between to 2 k-means clustering
according to Marina Meila, "The Uniqueness of a Good Optimum for K-Means", ICML 2006
Author : Diviyan Kalainathan
Date : 20/06/2016

'''

import csv,numpy,itertools
from sklearn import metrics

def Clustering_performance_evaluation(mode, folder_name, run1, run2, num_clusters, num_init):
    """
    :param mode: selects which metric is to be used
    :param folder_name: Folder of the runs (String)
    :param run1: Number of the run 1 (int)
    :param run2: Number of the run 2 (int)
    :param num_clusters:
    :param num_init:
    :return:  distance value (float?)

    """
    numpy.set_printoptions(threshold='nan')
    print('-'+str(num_clusters)+'---performance evaluation between runs : ' + str(run1) + ' ,' + str(run2))
    valid_data= True
    #Checking if the data is valid by loading & testing the shape of it
    try:
        data_1=numpy.loadtxt('output/'+folder_name+'/cluster_predictions_c'+ str(num_clusters)
                      + '_n'+ str(num_init) +'_r'+ str(run1)+'.csv',delimiter=';')
        data_2=numpy.loadtxt('output/'+folder_name+'/cluster_predictions_c'+ str(num_clusters)
                      + '_n'+ str(num_init) +'_r'+ str(run2)+'.csv',delimiter=';')

        if data_1.shape != data_2.shape:
            valid_data=False

    except IOError as e:
        print "I/O error({0}): {1}".format(e.errno, e.strerror)
        valid_data=False

    if valid_data:
        n_samples=data_1.shape[0]
        data_1 = numpy.asarray(sorted(data_1, key=lambda x: x[1]))
        data_2 = numpy.asarray(sorted(data_2, key=lambda x: x[1]))
        if mode==1:
            #Distance defined by Marina Meila : k! complexity
            clustering_1=numpy.zeros((n_samples,num_clusters))
            clustering_2=numpy.zeros((n_samples,num_clusters))

            for x in range(0,n_samples):
                clustering_1[x,data_1[x,0]]+=1
                clustering_2[x,data_2[x,0]]+=1

            '''for y in range(0,num_clusters):
                try:
                    clustering_1[:,y]*=1/numpy.sqrt(numpy.sum(clustering_1[:,y]))
                except ZeroDivisionError:
                    clustering_1[:,y]=0
                try:
                    clustering_2[:,y]*=1/numpy.sqrt(numpy.sum(clustering_2[:,y]))
                except ZeroDivisionError:
                    clustering_2[:,y]=0
            ''' # No normalisation needed

            confusion_matrix=numpy.dot(numpy.transpose(clustering_1),clustering_2)
            max_confusion=0
            result = []

            for perm in itertools.permutations(range(num_clusters)):
                confusion=0
                for i in range(0, num_clusters):
                    confusion += confusion_matrix[i, perm[i]]

                if max_confusion<confusion:
                    max_confusion=confusion

            distance=(max_confusion/n_samples)
            return distance

        elif mode==2:
            #Ajusted rand index
            distance=metrics.adjusted_rand_score(data_1[:,0],data_2[:,0])

            return distance

        elif mode==3:
            #V-mesure
            distance=metrics.v_measure_score(data_1[:,0],data_2[:,0])

            return distance

    return 0


#!/usr/bin/env python
##
##  See COPYING file distributed along with the ncanda-data-integration package
##  for the copyright and license terms
##
"""
xnat_scans_filter.py
======================
This script filters the csv file generated using xnat_extractor.py. This filters
is based on records from XNAT where there is one row per scan.

xnat_scans_filter.py -i path/to/xnat.csv
"""

import os
import sys

import redcap
import pandas as pd

# Fields to retrieve from REDCap
fields = ['study_id', 'redcap_event_name', 'exclude', 'visit_ignore',
          'visit_date', 'mri_missing', 'mri_xnat_sid', 'mri_xnat_eids',
          'visit_notes']

# Forms that the fields come from in REDCap.
forms = ['mr_session_report', 'visit_date', 'demographics']

def get_project_entry(args=None):
    """
    Pulls the data from REDCap
    """
    # Get API key.
    summary_key_file = open(os.path.join(os.path.expanduser("~"),
                                         '.server_config',
                                         'redcap-dataentry-token'), 'r')
    summary_api_key = summary_key_file.read().strip()

    # Connect to API.
    project_entry = redcap.Project('https://ncanda.sri.com/redcap/api/',
                                   summary_api_key,
                                   verify_ssl=False)
    return project_entry

def data_entry_fields(fields, project, arm):
    """
    Gets the dataframe containing a specific arm from REDCap
    """
    # Get a dataframe of fields
    data_entry_raw = project.export_records(fields=fields,
                                            forms=forms,
                                            format='df',
                                            events=[arm])
    return data_entry_raw

def append_site_id_row(xnat_df, scans_df):
    scans_df['site_id'] = ''
    ids = xnat_df[['site_id', 'subject_id']]
    map = {r.subject_id: r.site_id for idx, r in ids.iterrows()}
    for idx, row in scans_df.iterrows():
        scans_df.site_id.loc[idx] = map.get(row.case)
    return scans_df

def is_in_redcap(rc_df, scans_df):
    """
    Checks if the scans missing in the pipeline are listed in REDCap and adds
    a column indicating as such.
    """
    scans_df['in_redcap'] = False
    scans_df['visit_ignore___yes'] = ''
    scans_df['visit_ignore_why'] = ''
    scans_df['visit_ignore_why_other'] = ''
    scans_df['visit_notes'] = ''
    scans_df['mri_missing'] = ''
    scans_df['mri_missing_why'] = ''
    scans_df['mri_missing_why_other'] = ''
    scans_df['mri_notes'] = ''
    rc_cases = rc_df[rc_df.mri_xnat_sid.isin(scans_df.case)]
    for idx, row in rc_cases.iterrows():
        scan_cases = scans_df[scans_df.case == row.mri_xnat_sid]
        scans_df.in_redcap.loc[scan_cases.index] = True
        # Visit info
        scans_df.visit_ignore___yes.loc[scan_cases.index] = row.visit_ignore___yes
        scans_df.visit_ignore_why.loc[scan_cases.index] = row.visit_ignore_why
        scans_df.visit_ignore_why_other.loc[scan_cases.index] = row.visit_ignore_why_other
        scans_df.visit_notes.loc[scan_cases.index] = row.visit_notes
        # Scan info
        scans_df.mri_missing.loc[scan_cases.index] = row.mri_missing
        scans_df.mri_missing_why.loc[scan_cases.index] = row.mri_missing_why
        scans_df.mri_missing_why_other.loc[scan_cases.index] = row.mri_missing_why_other
        scans_df.mri_notes.loc[scan_cases.index] = row.mri_notes
    return scans_df

def is_in_xnat(xnat_df, scans_df):
    """
    Checks XNAT for scans near the visit date recorded in REDCap
    """

def main(args=None):
    # Connect to REDCap
    project_entry = get_project_entry()

    # Get the visit dataframe
    project_df = data_entry_fields(fields, project_entry, args.event)

    # Get a list of all EIDs for the given visit
    xnat_eids = project_df['mri_xnat_eids'].tolist()

    # Read the csv file from xnat_extractor
    xnat_csv = pd.read_csv(args.infile)

    # Filter the XNAT records by the EIDs in REDCap
    # This provides a list of all the scans in XNAT that are also in REDCap
    filter_csv = xnat_csv[xnat_csv['experiment_id'].isin(xnat_eids)]

    # Iterate through scans missing in the pipeline and check:
    #   - they are present in the filtered REDCap list
    # present in the REDCap filter list.
    if args.missing_scans:
        list_missing_scans = pd.read_csv(args.missing_scans)
        missing_scans_df = append_site_id_row(xnat_csv, list_missing_scans)

        # Add columns indicating if there is data for this visit in redcap
        in_redcap = is_in_redcap(project_df, missing_scans_df)
        filter_csv = in_redcap

    # Write the results to disk
    filter_csv.to_csv(args.outfile)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--infile',
                        required=True,
                        help="Input csv file from xnat_extractor.py")
    parser.add_argument('-e', '--event',
                        choices=['baseline_visit_arm_1', '1y_visit_arm_1'],
                        default='1y_visit_arm_1')
    parser.add_argument('-m', '--missing-scans',
                        help="Output of list_missing_scans script.")
    parser.add_argument('-o', '--outfile',
                        default=os.path.join('/tmp', 'xnat_scans_filter.csv'))
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='Enable verbose reporting.')
    argv = parser.parse_args()
    sys.exit(main(args=argv))