#!/usr/bin/env python
"""
Example of embedding matplotlib in an application and interacting with
a treeview to store data.  Double click on an entry to update plot
data

"""
import pygtk
pygtk.require('2.0')
import gtk
from gtk import gdk

import matplotlib
matplotlib.use('GTKAgg')  # or 'GTK'
from matplotlib.backends.backend_gtk import FigureCanvasGTK as FigureCanvas

from numpy.random import random
from matplotlib.figure import Figure

class DataManager(gtk.Window):
    numRows, numCols = 20,10

    data = random((numRows, numCols))

    def __init__(self):
        gtk.Window.__init__(self)
        self.set_default_size(600, 600)
        self.connect('destroy', lambda win: gtk.main_quit())

        self.set_title('GtkListStore demo')
        self.set_border_width(8)

        vbox = gtk.VBox(False, 8)
        self.add(vbox)

        label = gtk.Label('Double click a row to plot the data')

        vbox.pack_start(label, False, False)

        sw = gtk.ScrolledWindow()
        sw.set_shadow_type(gtk.SHADOW_ETCHED_IN)
        sw.set_policy(gtk.POLICY_NEVER,
                      gtk.POLICY_AUTOMATIC)
        vbox.pack_start(sw, True, True)

        model = self.create_model()

        self.treeview = gtk.TreeView(model)
        self.treeview.set_rules_hint(True)

        # matplotlib stuff
        fig = Figure(figsize=(6,4))

        self.canvas = FigureCanvas(fig)  # a gtk.DrawingArea
        vbox.pack_start(self.canvas, True, True)
        ax = fig.add_subplot(111)
        self.line, = ax.plot(self.data[0,:], 'go')  # plot the first row

        self.treeview.connect('row-activated', self.plot_row)
        sw.add(self.treeview)

        self.add_columns()

        self.add_events(gdk.BUTTON_PRESS_MASK |
                        gdk.KEY_PRESS_MASK|
                        gdk.KEY_RELEASE_MASK)

    def plot_row(self, treeview, path, view_column):
        ind, = path  # get the index into data
        points = self.data[ind,:]
        self.line.set_ydata(points)
        self.canvas.draw()

    def add_columns(self):
        for i in range(self.numCols):
            column = gtk.TreeViewColumn('%d'%i, gtk.CellRendererText(), text=i)
            self.treeview.append_column(column)

    def create_model(self):
        types = [float]*self.numCols
        store = gtk.ListStore(*types)

        for row in self.data:
            store.append(row)
        return store

manager = DataManager()
manager.show_all()
gtk.main()


# -*- coding: utf-8 -*-
"""
DBSCAN: Density-Based Spatial Clustering of Applications with Noise
"""

# Author: Robert Layton <robertlayton@gmail.com>
#         Joel Nothman <joel.nothman@gmail.com>
#         Lars Buitinck
#
# License: BSD 3 clause

import numpy as np
from scipy import sparse

from ..base import BaseEstimator, ClusterMixin
from ..metrics import pairwise_distances
from ..utils import check_array, check_consistent_length
from ..utils.fixes import astype
from ..neighbors import NearestNeighbors

from ._dbscan_inner import dbscan_inner

def dbscan(X, eps=0.5, min_samples=5, metric='minkowski',
           algorithm='auto', leaf_size=30, p=2, sample_weight=None):
    """Perform DBSCAN clustering from vector array or distance matrix.

    Read more in the :ref:`User Guide <dbscan>`.

    Parameters
    ----------
    X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
            array of shape (n_samples, n_samples)
        A feature array, or array of distances between samples if
        ``metric='precomputed'``.

    eps : float, optional
        The maximum distance between two samples for them to be considered
        as in the same neighborhood.

    min_samples : int, optional
        The number of samples (or total weight) in a neighborhood for a point
        to be considered as a core point. This includes the point itself.

    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.pairwise_distances for its
        metric parameter.
        If metric is "precomputed", X is assumed to be a distance matrix and
        must be square. X may be a sparse matrix, in which case only "nonzero"
        elements may be considered neighbors for DBSCAN.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        The algorithm to be used by the NearestNeighbors module
        to compute pointwise distances and find nearest neighbors.
        See NearestNeighbors module documentation for details.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or cKDTree. This can affect the speed
        of the construction and query, as well as the memory required
        to store the tree. The optimal value depends
        on the nature of the problem.

    p : float, optional
        The power of the Minkowski metric to be used to calculate distance
        between points.

    sample_weight : array, shape (n_samples,), optional
        Weight of each sample, such that a sample with a weight of at least
        ``min_samples`` is by itself a core sample; a sample with negative
        weight may inhibit its eps-neighbor from being core.
        Note that weights are absolute, and default to 1.

    Returns
    -------
    core_samples : array [n_core_samples]
        Indices of core samples.

    labels : array [n_samples]
        Cluster labels for each point.  Noisy samples are given the label -1.

    Notes
    -----
    See examples/cluster/plot_dbscan.py for an example.

    This implementation bulk-computes all neighborhood queries, which increases
    the memory complexity to O(n.d) where d is the average number of neighbors,
    while original DBSCAN had memory complexity O(n).

    Sparse neighborhoods can be precomputed using
    :func:`NearestNeighbors.radius_neighbors_graph
    <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>`
    with ``mode='distance'``.

    References
    ----------
    Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based
    Algorithm for Discovering Clusters in Large Spatial Databases with Noise".
    In: Proceedings of the 2nd International Conference on Knowledge Discovery
    and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
    """
    if not eps > 0.0:
        raise ValueError("eps must be positive.")

    X = check_array(X, accept_sparse='csr')
    if sample_weight is not None:
        sample_weight = np.asarray(sample_weight)
        check_consistent_length(X, sample_weight)

    # Calculate neighborhood for all samples. This leaves the original point
    # in, which needs to be considered later (i.e. point i is in the
    # neighborhood of point i. While True, its useless information)
    if metric == 'precomputed' and sparse.issparse(X):
        neighborhoods = np.empty(X.shape[0], dtype=object)
        X.sum_duplicates()  # XXX: modifies X's internals in-place
        X_mask = X.data <= eps
        masked_indices = astype(X.indices, np.intp, copy=False)[X_mask]
        masked_indptr = np.cumsum(X_mask)[X.indptr[1:] - 1]
        # insert the diagonal: a point is its own neighbor, but 0 distance
        # means absence from sparse matrix data
        masked_indices = np.insert(masked_indices, masked_indptr,
                                   np.arange(X.shape[0]))
        masked_indptr = masked_indptr[:-1] + np.arange(1, X.shape[0])
        # split into rows
        neighborhoods[:] = np.split(masked_indices, masked_indptr)
    else:
        neighbors_model = NearestNeighbors(radius=eps, algorithm=algorithm,
                                           leaf_size=leaf_size,
                                           metric=metric, p=p)
        neighbors_model.fit(X)
        # This has worst case O(n^2) memory complexity
        neighborhoods = neighbors_model.radius_neighbors(X, eps,
                                                         return_distance=False)

    if sample_weight is None:
        n_neighbors = np.array([len(neighbors)
                                for neighbors in neighborhoods])
    else:
        n_neighbors = np.array([np.sum(sample_weight[neighbors])
                                for neighbors in neighborhoods])

    # Initially, all samples are noise.
    labels = -np.ones(X.shape[0], dtype=np.intp)

    # A list of all core samples found.
    core_samples = np.asarray(n_neighbors >= min_samples, dtype=np.uint8)
    dbscan_inner(core_samples, neighborhoods, labels)
    return np.where(core_samples)[0], labels

class DBSCAN(BaseEstimator, ClusterMixin):
    """Perform DBSCAN clustering from vector array or distance matrix.

    DBSCAN - Density-Based Spatial Clustering of Applications with Noise.
    Finds core samples of high density and expands clusters from them.
    Good for data which contains clusters of similar density.

    Read more in the :ref:`User Guide <dbscan>`.

    Parameters
    ----------
    eps : float, optional
        The maximum distance between two samples for them to be considered
        as in the same neighborhood.
    min_samples : int, optional
        The number of samples (or total weight) in a neighborhood for a point
        to be considered as a core point. This includes the point itself.
    metric : string, or callable
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string or callable, it must be one of
        the options allowed by metrics.pairwise.calculate_distance for its
        metric parameter.
        If metric is "precomputed", X is assumed to be a distance matrix and
        must be square. X may be a sparse matrix, in which case only "nonzero"
        elements may be considered neighbors for DBSCAN.
    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        The algorithm to be used by the NearestNeighbors module
        to compute pointwise distances and find nearest neighbors.
        See NearestNeighbors module documentation for details.
    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or cKDTree. This can affect the speed
        of the construction and query, as well as the memory required
        to store the tree. The optimal value depends
        on the nature of the problem.

    Attributes
    ----------
    core_sample_indices_ : array, shape = [n_core_samples]
        Indices of core samples.

    components_ : array, shape = [n_core_samples, n_features]
        Copy of each core sample found by training.

    labels_ : array, shape = [n_samples]
        Cluster labels for each point in the dataset given to fit().
        Noisy samples are given the label -1.

    Notes
    -----
    See examples/cluster/plot_dbscan.py for an example.

    This implementation bulk-computes all neighborhood queries, which increases
    the memory complexity to O(n.d) where d is the average number of neighbors,
    while original DBSCAN had memory complexity O(n).

    Sparse neighborhoods can be precomputed using
    :func:`NearestNeighbors.radius_neighbors_graph
    <sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>`
    with ``mode='distance'``.

    References
    ----------
    Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based
    Algorithm for Discovering Clusters in Large Spatial Databases with Noise".
    In: Proceedings of the 2nd International Conference on Knowledge Discovery
    and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
    """

    def __init__(self, eps=0.5, min_samples=5, metric='euclidean',
                 algorithm='auto', leaf_size=30, p=None):
        self.eps = eps
        self.min_samples = min_samples
        self.metric = metric
        self.algorithm = algorithm
        self.leaf_size = leaf_size
        self.p = p

    def fit(self, X, y=None, sample_weight=None):
        """Perform DBSCAN clustering from features or distance matrix.

        Parameters
        ----------
        X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
                array of shape (n_samples, n_samples)
            A feature array, or array of distances between samples if
            ``metric='precomputed'``.
        sample_weight : array, shape (n_samples,), optional
            Weight of each sample, such that a sample with a weight of at least
            ``min_samples`` is by itself a core sample; a sample with negative
            weight may inhibit its eps-neighbor from being core.
            Note that weights are absolute, and default to 1.
        """
        X = check_array(X, accept_sparse='csr')
        clust = dbscan(X, sample_weight=sample_weight, **self.get_params())
        self.core_sample_indices_, self.labels_ = clust
        if len(self.core_sample_indices_):
            # fix for scipy sparse indexing issue
            self.components_ = X[self.core_sample_indices_].copy()
        else:
            # no core samples
            self.components_ = np.empty((0, X.shape[1]))
        return self

    def fit_predict(self, X, y=None, sample_weight=None):
        """Performs clustering on X and returns cluster labels.

        Parameters
        ----------
        X : array or sparse (CSR) matrix of shape (n_samples, n_features), or \
                array of shape (n_samples, n_samples)
            A feature array, or array of distances between samples if
            ``metric='precomputed'``.
        sample_weight : array, shape (n_samples,), optional
            Weight of each sample, such that a sample with a weight of at least
            ``min_samples`` is by itself a core sample; a sample with negative
            weight may inhibit its eps-neighbor from being core.
            Note that weights are absolute, and default to 1.

        Returns
        -------
        y : ndarray, shape (n_samples,)
            cluster labels
        """
        self.fit(X, sample_weight=sample_weight)
        return self.labels_


import os
import pandas as pd
import numpy as np
import pytest
from astropy import units as u
from numpy.testing import assert_almost_equal

###
# Save and Load
###

@pytest.fixture(scope="module", autouse=True)
def to_hdf_buffer(hdf_file_path, simulation_verysimple):
    simulation_verysimple.runner.to_hdf(hdf_file_path, name='runner')

runner_properties = ['output_nu', 'output_energy', 'nu_bar_estimator',
                     'j_estimator', 'montecarlo_virtual_luminosity',
                     'last_interaction_in_nu',
                     'last_interaction_type',
                     'last_line_interaction_in_id',
                     'last_line_interaction_out_id',
                     'last_line_interaction_shell_id',
                     'packet_luminosity']

@pytest.mark.parametrize("attr", runner_properties)
def test_hdf_runner(hdf_file_path, simulation_verysimple, attr):
    actual = getattr(simulation_verysimple.runner, attr)
    if hasattr(actual, 'cgs'):
        actual = actual.cgs.value
    path = os.path.join('runner', attr)
    expected = pd.read_hdf(hdf_file_path, path)
    assert_almost_equal(actual, expected.values)


import os
import pandas as pd

from dataactbroker.scripts import update_historical_duns
from dataactcore.config import CONFIG_BROKER
from dataactcore.utils.duns import DUNS_COLUMNS, EXCLUDE_FROM_API
from dataactcore.models.domainModels import DUNS, HistoricDUNS

def test_remove_existing_duns(database):
    """ Testing the removing existing duns function"""
    sess = database.session
    # of the duns 000000001-000000009, half of them are in the database
    all_duns = ['00000000{}'.format(x) for x in range(0, 10)]
    existing_duns = all_duns[: 4]
    data = pd.DataFrame.from_dict({'awardee_or_recipient_uniqu': all_duns})
    for duns in existing_duns:
        sess.add(DUNS(awardee_or_recipient_uniqu=duns))
    sess.commit()

    # confirm that the dataframe returned only has half the duns
    expected_duns = list(set(existing_duns) ^ set(all_duns))
    new_df = update_historical_duns.remove_existing_duns(data, sess)
    assert sorted(expected_duns) == sorted(new_df['awardee_or_recipient_uniqu'].tolist())

def mock_get_duns_props_from_sam(duns_list):
    """ Mock function for get_duns_props as we can't connect to the SAM service """
    request_cols = [col for col in DUNS_COLUMNS if col not in EXCLUDE_FROM_API]
    columns = request_cols
    results = pd.DataFrame(columns=columns)

    duns_mappings = {
        '000000001': {
            'awardee_or_recipient_uniqu': '000000001',
            'uei': 'A1',
            'legal_business_name': 'Legal Name 1',
            'dba_name': 'Name 1',
            'entity_structure': '1A',
            'ultimate_parent_unique_ide': '999999999',
            'ultimate_parent_uei': 'Z9',
            'ultimate_parent_legal_enti': 'Parent Legal Name 1',
            'address_line_1': 'Test address 1',
            'address_line_2': 'Test address 2',
            'city': 'Test city',
            'state': 'Test state',
            'zip': 'Test zip',
            'zip4': 'Test zip4',
            'country_code': 'Test country',
            'congressional_district': 'Test congressional district',
            'business_types_codes': [['A', 'B', 'C']],
            'business_types': [['Name A', 'Name B', 'Name C']],
            'high_comp_officer1_full_na': 'Test Exec 1',
            'high_comp_officer1_amount': '1',
            'high_comp_officer2_full_na': 'Test Exec 2',
            'high_comp_officer2_amount': '2',
            'high_comp_officer3_full_na': 'Test Exec 3',
            'high_comp_officer3_amount': '3',
            'high_comp_officer4_full_na': 'Test Exec 4',
            'high_comp_officer4_amount': '4',
            'high_comp_officer5_full_na': 'Test Exec 5',
            'high_comp_officer5_amount': '5'
        },
        '000000002': {
            'awardee_or_recipient_uniqu': '000000002',
            'uei': 'B2',
            'legal_business_name': 'Legal Name 2',
            'dba_name': 'Name 2',
            'entity_structure': '2B',
            'ultimate_parent_unique_ide': '999999998',
            'ultimate_parent_uei': 'Y8',
            'ultimate_parent_legal_enti': 'Parent Legal Name 2',
            'address_line_1': 'Other Test address 1',
            'address_line_2': 'Other Test address 2',
            'city': 'Other Test city',
            'state': 'Other Test state',
            'zip': 'Other Test zip',
            'zip4': 'Other Test zip4',
            'country_code': 'Other Test country',
            'congressional_district': 'Other Test congressional district',
            'business_types_codes': [['D', 'E', 'F']],
            'business_types': [['Name D', 'Name E', 'Name F']],
            'high_comp_officer1_full_na': 'Test Other Exec 6',
            'high_comp_officer1_amount': '6',
            'high_comp_officer2_full_na': 'Test Other Exec 7',
            'high_comp_officer2_amount': '7',
            'high_comp_officer3_full_na': 'Test Other Exec 8',
            'high_comp_officer3_amount': '8',
            'high_comp_officer4_full_na': 'Test Other Exec 9',
            'high_comp_officer4_amount': '9',
            'high_comp_officer5_full_na': 'Test Other Exec 10',
            'high_comp_officer5_amount': '10'
        }
    }
    for duns in duns_list:
        if duns in duns_mappings:
            results = results.append(pd.DataFrame(duns_mappings[duns]), sort=True)
    return results

def test_update_duns_props(monkeypatch):
    """ Testing updating the duns props with both populated/blank data """
    monkeypatch.setattr('dataactcore.utils.duns.get_duns_props_from_sam', mock_get_duns_props_from_sam)
    duns_df = pd.DataFrame.from_dict({
        'awardee_or_recipient_uniqu': ['000000001', '000000002', '000000003']
    })

    expected_df = pd.DataFrame.from_dict({
        'awardee_or_recipient_uniqu': ['000000001', '000000002', '000000003'],
        'uei': ['A1', 'B2', None],
        'address_line_1': ['Test address 1', 'Other Test address 1', None],
        'address_line_2': ['Test address 2', 'Other Test address 2', None],
        'city': ['Test city', 'Other Test city', None],
        'state': ['Test state', 'Other Test state', None],
        'zip': ['Test zip', 'Other Test zip', None],
        'zip4': ['Test zip4', 'Other Test zip4', None],
        'country_code': ['Test country', 'Other Test country', None],
        'congressional_district': ['Test congressional district', 'Other Test congressional district', None],
        'business_types_codes': [['A', 'B', 'C'], ['D', 'E', 'F'], []],
        'business_types': [['Name A', 'Name B', 'Name C'], ['Name D', 'Name E', 'Name F'], []],
        'entity_structure': ['1A', '2B', None],
        'dba_name': ['Name 1', 'Name 2', None],
        'ultimate_parent_unique_ide': ['999999999', '999999998', None],
        'ultimate_parent_uei': ['Z9', 'Y8', None],
        'ultimate_parent_legal_enti': ['Parent Legal Name 1', 'Parent Legal Name 2', None],
        'high_comp_officer1_full_na': ['Test Exec 1', 'Test Other Exec 6', None],
        'high_comp_officer1_amount': ['1', '6', None],
        'high_comp_officer2_full_na': ['Test Exec 1', 'Test Other Exec 7', None],
        'high_comp_officer2_amount': ['2', '7', None],
        'high_comp_officer3_full_na': ['Test Exec 1', 'Test Other Exec 8', None],
        'high_comp_officer3_amount': ['3', '8', None],
        'high_comp_officer4_full_na': ['Test Exec 1', 'Test Other Exec 9', None],
        'high_comp_officer4_amount': ['4', '9', None],
        'high_comp_officer5_full_na': ['Test Exec 1', 'Test Other Exec 10', None],
        'high_comp_officer5_amount': ['5', '10', None]
    })

    assert expected_df.sort_index(inplace=True) == update_historical_duns.update_duns_props(duns_df)\
        .sort_index(inplace=True)

def test_update_duns_props_empty(monkeypatch):
    """ Special case where no data is returned """
    monkeypatch.setattr('dataactcore.utils.duns.get_duns_props_from_sam', mock_get_duns_props_from_sam)
    duns_df = pd.DataFrame.from_dict({
        'awardee_or_recipient_uniqu': ['000000003']
    })

    expected_df = pd.DataFrame.from_dict({
        'awardee_or_recipient_uniqu': ['000000003'],
        'uei': [None],
        'address_line_1': [None],
        'address_line_2': [None],
        'city': [None],
        'state': [None],
        'zip': [None],
        'zip4': [None],
        'country_code': [None],
        'congressional_district': [None],
        'business_types_codes': [[]],
        'business_types': [[]],
        'dba_name': [None],
        'entity_structure': [None],
        'ultimate_parent_unique_ide': [None],
        'ultimate_parent_uei': [None],
        'ultimate_parent_legal_enti': [None],
        'high_comp_officer1_full_na': [None],
        'high_comp_officer1_amount': [None],
        'high_comp_officer2_full_na': [None],
        'high_comp_officer2_amount': [None],
        'high_comp_officer3_full_na': [None],
        'high_comp_officer3_amount': [None],
        'high_comp_officer4_full_na': [None],
        'high_comp_officer4_amount': [None],
        'high_comp_officer5_full_na': [None],
        'high_comp_officer5_amount': [None]
    })

    assert expected_df.to_dict() == update_historical_duns.update_duns_props(duns_df).to_dict()

def test_run_duns_batches(database, monkeypatch):
    """ Test run_duns_batches for the core functionality """
    monkeypatch.setattr('dataactcore.utils.duns.get_duns_props_from_sam', mock_get_duns_props_from_sam)
    sess = database.session
    all_duns = ['00000000{}'.format(x) for x in range(1, 5)]
    existing_duns = all_duns[2:]
    for duns in existing_duns:
        sess.add(DUNS(awardee_or_recipient_uniqu=duns))
    sess.commit()

    duns_file = os.path.join(CONFIG_BROKER['path'], 'tests', 'unit', 'data', 'historic_DUNS_export_small.csv')

    update_historical_duns.run_duns_batches(duns_file, sess, block_size=1)

    expected_results = {
        '000000001': {
            'awardee_or_recipient_uniqu': '000000001',
            'uei': 'A1',
            'registration_date': '2004-04-01',
            'expiration_date': '2013-01-11',
            'last_sam_mod_date': '2013-01-11',
            'activation_date': '2012-01-11',
            'legal_business_name': 'TEST DUNS 1',
            'address_line_1': 'Test address 1',
            'address_line_2': 'Test address 2',
            'city': 'Test city',
            'state': 'Test state',
            'zip': 'Test zip',
            'zip4': 'Test zip4',
            'country_code': 'Test country',
            'congressional_district': 'Test congressional district',
            'business_types_codes': ['A', 'B', 'C'],
            'business_types': ['Name A', 'Name B', 'Name C'],
            'dba_name': 'Name 1',
            'entity_structure': '1A',
            'ultimate_parent_unique_ide': '999999999',
            'ultimate_parent_uei': 'Z9',
            'ultimate_parent_legal_enti': 'Parent Legal Name 1',
            'high_comp_officer1_full_na': 'Test Exec 1',
            'high_comp_officer1_amount': '1',
            'high_comp_officer2_full_na': 'Test Exec 2',
            'high_comp_officer2_amount': '2',
            'high_comp_officer3_full_na': 'Test Exec 3',
            'high_comp_officer3_amount': '3',
            'high_comp_officer4_full_na': 'Test Exec 4',
            'high_comp_officer4_amount': '4',
            'high_comp_officer5_full_na': 'Test Exec 5',
            'high_comp_officer5_amount': '5'
        },
        '000000002': {
            'awardee_or_recipient_uniqu': '000000002',
            'uei': 'B2',
            'registration_date': '2004-04-02',
            'expiration_date': '2013-01-12',
            'last_sam_mod_date': '2013-01-12',
            'activation_date': '2012-01-12',
            'legal_business_name': 'TEST DUNS 2',
            'address_line_1': 'Other Test address 1',
            'address_line_2': 'Other Test address 2',
            'city': 'Other Test city',
            'state': 'Other Test state',
            'zip': 'Other Test zip',
            'zip4': 'Other Test zip4',
            'country_code': 'Other Test country',
            'congressional_district': 'Other Test congressional district',
            'business_types_codes': ['D', 'E', 'F'],
            'business_types': ['Name D', 'Name E', 'Name F'],
            'dba_name': 'Name 2',
            'entity_structure': '2B',
            'ultimate_parent_unique_ide': '999999998',
            'ultimate_parent_uei': 'Y8',
            'ultimate_parent_legal_enti': 'Parent Legal Name 2',
            'high_comp_officer1_full_na': 'Test Other Exec 6',
            'high_comp_officer1_amount': '6',
            'high_comp_officer2_full_na': 'Test Other Exec 7',
            'high_comp_officer2_amount': '7',
            'high_comp_officer3_full_na': 'Test Other Exec 8',
            'high_comp_officer3_amount': '8',
            'high_comp_officer4_full_na': 'Test Other Exec 9',
            'high_comp_officer4_amount': '9',
            'high_comp_officer5_full_na': 'Test Other Exec 10',
            'high_comp_officer5_amount': '10'
        }
    }
    results = {}
    for duns_obj in sess.query(HistoricDUNS).all():
        results[duns_obj.awardee_or_recipient_uniqu] = {
            'awardee_or_recipient_uniqu': duns_obj.awardee_or_recipient_uniqu,
            'uei': duns_obj.uei,
            'registration_date': str(duns_obj.registration_date) if duns_obj.registration_date else None,
            'expiration_date': str(duns_obj.expiration_date) if duns_obj.expiration_date else None,
            'last_sam_mod_date': str(duns_obj.last_sam_mod_date) if duns_obj.last_sam_mod_date else None,
            'activation_date': str(duns_obj.activation_date) if duns_obj.activation_date else None,
            'legal_business_name': duns_obj.legal_business_name,
            'address_line_1': duns_obj.address_line_1,
            'address_line_2': duns_obj.address_line_2,
            'city': duns_obj.city,
            'state': duns_obj.state,
            'zip': duns_obj.zip,
            'zip4': duns_obj.zip4,
            'country_code': duns_obj.country_code,
            'congressional_district': duns_obj.congressional_district,
            'business_types_codes': duns_obj.business_types_codes,
            'business_types': duns_obj.business_types,
            'dba_name': duns_obj.dba_name,
            'entity_structure': duns_obj.entity_structure,
            'ultimate_parent_unique_ide': duns_obj.ultimate_parent_unique_ide,
            'ultimate_parent_uei': duns_obj.ultimate_parent_uei,
            'ultimate_parent_legal_enti': duns_obj.ultimate_parent_legal_enti,
            'high_comp_officer1_full_na': duns_obj.high_comp_officer1_full_na,
            'high_comp_officer1_amount': duns_obj.high_comp_officer1_amount,
            'high_comp_officer2_full_na': duns_obj.high_comp_officer2_full_na,
            'high_comp_officer2_amount': duns_obj.high_comp_officer2_amount,
            'high_comp_officer3_full_na': duns_obj.high_comp_officer3_full_na,
            'high_comp_officer3_amount': duns_obj.high_comp_officer3_amount,
            'high_comp_officer4_full_na': duns_obj.high_comp_officer4_full_na,
            'high_comp_officer4_amount': duns_obj.high_comp_officer4_amount,
            'high_comp_officer5_full_na': duns_obj.high_comp_officer5_full_na,
            'high_comp_officer5_amount': duns_obj.high_comp_officer5_amount
        }
    assert results == expected_results

def test_workflows(database, monkeypatch):
    """ Test both scenarios of the script, starting with a full run """
    monkeypatch.setattr('dataactcore.utils.duns.get_duns_props_from_sam', mock_get_duns_props_from_sam)
    sess = database.session
    all_duns = ['00000000{}'.format(x) for x in range(1, 5)]
    existing_duns = all_duns[2:]
    for duns in existing_duns:
        sess.add(DUNS(awardee_or_recipient_uniqu=duns))
    sess.commit()

    duns_file = os.path.join(CONFIG_BROKER['path'], 'tests', 'unit', 'data', 'historic_DUNS_export_small.csv')

    update_historical_duns.run_duns_batches(duns_file, sess, block_size=1)
    update_historical_duns.import_historic_duns(sess)

    expected_results = {
        '000000001': {
            'awardee_or_recipient_uniqu': '000000001',
            'uei': 'A1',
            'registration_date': '2004-04-01',
            'expiration_date': '2013-01-11',
            'last_sam_mod_date': '2013-01-11',
            'activation_date': '2012-01-11',
            'legal_business_name': 'TEST DUNS 1',
            'address_line_1': 'Test address 1',
            'address_line_2': 'Test address 2',
            'city': 'Test city',
            'state': 'Test state',
            'zip': 'Test zip',
            'zip4': 'Test zip4',
            'country_code': 'Test country',
            'congressional_district': 'Test congressional district',
            'business_types_codes': ['A', 'B', 'C'],
            'business_types': ['Name A', 'Name B', 'Name C'],
            'dba_name': 'Name 1',
            'entity_structure': '1A',
            'ultimate_parent_unique_ide': '999999999',
            'ultimate_parent_uei': 'Z9',
            'ultimate_parent_legal_enti': 'Parent Legal Name 1',
            'high_comp_officer1_full_na': 'Test Exec 1',
            'high_comp_officer1_amount': '1',
            'high_comp_officer2_full_na': 'Test Exec 2',
            'high_comp_officer2_amount': '2',
            'high_comp_officer3_full_na': 'Test Exec 3',
            'high_comp_officer3_amount': '3',
            'high_comp_officer4_full_na': 'Test Exec 4',
            'high_comp_officer4_amount': '4',
            'high_comp_officer5_full_na': 'Test Exec 5',
            'high_comp_officer5_amount': '5'
        },
        '000000002': {
            'awardee_or_recipient_uniqu': '000000002',
            'uei': 'B2',
            'registration_date': '2004-04-02',
            'expiration_date': '2013-01-12',
            'last_sam_mod_date': '2013-01-12',
            'activation_date': '2012-01-12',
            'legal_business_name': 'TEST DUNS 2',
            'address_line_1': 'Other Test address 1',
            'address_line_2': 'Other Test address 2',
            'city': 'Other Test city',
            'state': 'Other Test state',
            'zip': 'Other Test zip',
            'zip4': 'Other Test zip4',
            'country_code': 'Other Test country',
            'congressional_district': 'Other Test congressional district',
            'business_types_codes': ['D', 'E', 'F'],
            'business_types': ['Name D', 'Name E', 'Name F'],
            'dba_name': 'Name 2',
            'entity_structure': '2B',
            'ultimate_parent_unique_ide': '999999998',
            'ultimate_parent_uei': 'Y8',
            'ultimate_parent_legal_enti': 'Parent Legal Name 2',
            'high_comp_officer1_full_na': 'Test Other Exec 6',
            'high_comp_officer1_amount': '6',
            'high_comp_officer2_full_na': 'Test Other Exec 7',
            'high_comp_officer2_amount': '7',
            'high_comp_officer3_full_na': 'Test Other Exec 8',
            'high_comp_officer3_amount': '8',
            'high_comp_officer4_full_na': 'Test Other Exec 9',
            'high_comp_officer4_amount': '9',
            'high_comp_officer5_full_na': 'Test Other Exec 10',
            'high_comp_officer5_amount': '10'
        },
        '000000003': {
            'awardee_or_recipient_uniqu': '000000003',
            'uei': None,
            'registration_date': None,
            'expiration_date': None,
            'last_sam_mod_date': None,
            'activation_date': None,
            'legal_business_name': None,
            'address_line_1': None,
            'address_line_2': None,
            'city': None,
            'state': None,
            'zip': None,
            'zip4': None,
            'country_code': None,
            'congressional_district': None,
            'business_types_codes': None,
            'business_types': None,
            'dba_name': None,
            'entity_structure': None,
            'ultimate_parent_unique_ide': None,
            'ultimate_parent_uei': None,
            'ultimate_parent_legal_enti': None,
            'high_comp_officer1_full_na': None,
            'high_comp_officer1_amount': None,
            'high_comp_officer2_full_na': None,
            'high_comp_officer2_amount': None,
            'high_comp_officer3_full_na': None,
            'high_comp_officer3_amount': None,
            'high_comp_officer4_full_na': None,
            'high_comp_officer4_amount': None,
            'high_comp_officer5_full_na': None,
            'high_comp_officer5_amount': None
        },
        '000000004': {
            'awardee_or_recipient_uniqu': '000000004',
            'uei': None,
            'registration_date': None,
            'expiration_date': None,
            'last_sam_mod_date': None,
            'activation_date': None,
            'legal_business_name': None,
            'address_line_1': None,
            'address_line_2': None,
            'city': None,
            'state': None,
            'zip': None,
            'zip4': None,
            'country_code': None,
            'congressional_district': None,
            'business_types_codes': None,
            'business_types': None,
            'dba_name': None,
            'entity_structure': None,
            'ultimate_parent_unique_ide': None,
            'ultimate_parent_uei': None,
            'ultimate_parent_legal_enti': None,
            'high_comp_officer1_full_na': None,
            'high_comp_officer1_amount': None,
            'high_comp_officer2_full_na': None,
            'high_comp_officer2_amount': None,
            'high_comp_officer3_full_na': None,
            'high_comp_officer3_amount': None,
            'high_comp_officer4_full_na': None,
            'high_comp_officer4_amount': None,
            'high_comp_officer5_full_na': None,
            'high_comp_officer5_amount': None
        }
    }
    results = {}
    for duns_obj in sess.query(DUNS).all():
        results[duns_obj.awardee_or_recipient_uniqu] = {
            'awardee_or_recipient_uniqu': duns_obj.awardee_or_recipient_uniqu,
            'uei': duns_obj.uei,
            'registration_date': str(duns_obj.registration_date) if duns_obj.registration_date else None,
            'expiration_date': str(duns_obj.expiration_date) if duns_obj.expiration_date else None,
            'last_sam_mod_date': str(duns_obj.last_sam_mod_date) if duns_obj.last_sam_mod_date else None,
            'activation_date': str(duns_obj.activation_date) if duns_obj.activation_date else None,
            'legal_business_name': duns_obj.legal_business_name,
            'address_line_1': duns_obj.address_line_1,
            'address_line_2': duns_obj.address_line_2,
            'city': duns_obj.city,
            'state': duns_obj.state,
            'zip': duns_obj.zip,
            'zip4': duns_obj.zip4,
            'country_code': duns_obj.country_code,
            'congressional_district': duns_obj.congressional_district,
            'business_types_codes': duns_obj.business_types_codes,
            'business_types': duns_obj.business_types,
            'dba_name': duns_obj.dba_name,
            'entity_structure': duns_obj.entity_structure,
            'ultimate_parent_unique_ide': duns_obj.ultimate_parent_unique_ide,
            'ultimate_parent_uei': duns_obj.ultimate_parent_uei,
            'ultimate_parent_legal_enti': duns_obj.ultimate_parent_legal_enti,
            'high_comp_officer1_full_na': duns_obj.high_comp_officer1_full_na,
            'high_comp_officer1_amount': duns_obj.high_comp_officer1_amount,
            'high_comp_officer2_full_na': duns_obj.high_comp_officer2_full_na,
            'high_comp_officer2_amount': duns_obj.high_comp_officer2_amount,
            'high_comp_officer3_full_na': duns_obj.high_comp_officer3_full_na,
            'high_comp_officer3_amount': duns_obj.high_comp_officer3_amount,
            'high_comp_officer4_full_na': duns_obj.high_comp_officer4_full_na,
            'high_comp_officer4_amount': duns_obj.high_comp_officer4_amount,
            'high_comp_officer5_full_na': duns_obj.high_comp_officer5_full_na,
            'high_comp_officer5_amount': duns_obj.high_comp_officer5_amount
        }
    assert results == expected_results

    # Test to see if truncating the DUNS table while keeping the historic reuploads the historic values
    sess.query(DUNS).filter(DUNS.historic.is_(True)).delete(synchronize_session=False)

    # Make sure all the historic DUNS are removed from the DUNS table
    assert sess.query(DUNS).filter(DUNS.historic.is_(True)).all() == []

    # Redo script but don't go through run_duns_batches
    update_historical_duns.clean_historic_duns(sess)
    update_historical_duns.import_historic_duns(sess)

    results = {}
    for duns_obj in sess.query(DUNS).all():
        results[duns_obj.awardee_or_recipient_uniqu] = {
            'awardee_or_recipient_uniqu': duns_obj.awardee_or_recipient_uniqu,
            'uei': duns_obj.uei,
            'registration_date': str(duns_obj.registration_date) if duns_obj.registration_date else None,
            'expiration_date': str(duns_obj.expiration_date) if duns_obj.expiration_date else None,
            'last_sam_mod_date': str(duns_obj.last_sam_mod_date) if duns_obj.last_sam_mod_date else None,
            'activation_date': str(duns_obj.activation_date) if duns_obj.activation_date else None,
            'legal_business_name': duns_obj.legal_business_name,
            'address_line_1': duns_obj.address_line_1,
            'address_line_2': duns_obj.address_line_2,
            'city': duns_obj.city,
            'state': duns_obj.state,
            'zip': duns_obj.zip,
            'zip4': duns_obj.zip4,
            'country_code': duns_obj.country_code,
            'congressional_district': duns_obj.congressional_district,
            'business_types_codes': duns_obj.business_types_codes,
            'business_types': duns_obj.business_types,
            'dba_name': duns_obj.dba_name,
            'entity_structure': duns_obj.entity_structure,
            'ultimate_parent_unique_ide': duns_obj.ultimate_parent_unique_ide,
            'ultimate_parent_uei': duns_obj.ultimate_parent_uei,
            'ultimate_parent_legal_enti': duns_obj.ultimate_parent_legal_enti,
            'high_comp_officer1_full_na': duns_obj.high_comp_officer1_full_na,
            'high_comp_officer1_amount': duns_obj.high_comp_officer1_amount,
            'high_comp_officer2_full_na': duns_obj.high_comp_officer2_full_na,
            'high_comp_officer2_amount': duns_obj.high_comp_officer2_amount,
            'high_comp_officer3_full_na': duns_obj.high_comp_officer3_full_na,
            'high_comp_officer3_amount': duns_obj.high_comp_officer3_amount,
            'high_comp_officer4_full_na': duns_obj.high_comp_officer4_full_na,
            'high_comp_officer4_amount': duns_obj.high_comp_officer4_amount,
            'high_comp_officer5_full_na': duns_obj.high_comp_officer5_full_na,
            'high_comp_officer5_amount': duns_obj.high_comp_officer5_amount
        }
    assert results == expected_results

def test_clean_historic_duns(database, monkeypatch):
    """
        Test to make sure if a new DUNS is loaded and we reload historic DUNS (skipping the major load),
        we should remove the historic equivalents.
    """
    monkeypatch.setattr('dataactcore.utils.duns.get_duns_props_from_sam', mock_get_duns_props_from_sam)
    sess = database.session
    all_duns = ['00000000{}'.format(x) for x in range(1, 5)]
    existing_duns = all_duns[2:]
    for duns in existing_duns:
        sess.add(DUNS(awardee_or_recipient_uniqu=duns))
    sess.commit()

    duns_file = os.path.join(CONFIG_BROKER['path'], 'tests', 'unit', 'data', 'historic_DUNS_export_small.csv')

    # normal run
    update_historical_duns.run_duns_batches(duns_file, sess, block_size=1)
    update_historical_duns.import_historic_duns(sess)

    # update old DUNS as part of load_duns_exec_comp.py
    updated_duns = sess.query(DUNS).filter(DUNS.awardee_or_recipient_uniqu == '000000002').one()
    updated_duns.historic = False
    sess.commit()

    # rerun with a skip
    update_historical_duns.clean_historic_duns(sess)
    update_historical_duns.import_historic_duns(sess)

    # check to see if historic duns equivalent is removed
    expected_count = sess.query(HistoricDUNS).filter(HistoricDUNS.awardee_or_recipient_uniqu == '000000002').count()
    assert expected_count == 0


#!/usr/bin/python

import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import numpy as np
import pandas as pd     # used to convert datetime64 to datetime
import csv
import sys
from mpl_toolkits.axes_grid1 import host_subplot
import mpl_toolkits.axisartist as AA

class Gen_Graph:
    def __init__(self, _filename):
        self.filename = _filename
        self.data = []
        self.dtype = []

    def readData(self):
        '''Read the data from .csv file'''
        with open(self.filename, 'r') as file:
            reader = csv.reader(file)
            for row in reader:
                self.data.append(tuple(row))
        return self.data

    def genDtype(self):
        '''Get the data type, always put DATE in the last '''
        for i in xrange(len(self.data[0])):
            if i != len(self.data[0]) - 1:
                self.dtype.append((str(self.data[0][i]), '<f8'))
            else:
                self.dtype.append((self.data[0][i], '<M8[s]'))

        print "Data Type: " + str(self.dtype)
        print '=============================================================='

    def uniqueish_color(self):
        '''Randomly select a color'''
        return plt.cm.gist_ncar(np.random.random())

    def genGraph(self):
        '''Generate the graph with unique y axis'''
        self.genDtype()
        x = np.array(self.data[1:], dtype=self.dtype)
        np.save('./graph_data/data', x)
        t = np.load('./graph_data/data.npy').view(np.recarray)
        fig, ax = plt.subplots(1)

        '''Drawing multiple lines in one graph'''
        for label in self.data[0]:
            if label != 'Time':
                dtype = t['{0}'.format(label)]
                ax.plot(pd.to_datetime(t.Time), dtype)
        ax.set_xlabel(' date ')

        '''Formatting the date'''
        fig.autofmt_xdate()
        ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d')
        plt.title('Sensor Data Flow')

        '''Create the labels for different lines'''
        labels = list(self.data[0][:-1])
        plt.legend(labels, loc='lower left')

        plt.show()

    def genGraph_m(self):
        '''Generate the graph with multiple y axises'''
        self.genDtype()
        x = np.array(self.data[1:], dtype=self.dtype)
        np.save('./graph_data/data', x)
        t = np.load('./graph_data/data.npy').view(np.recarray)
        fig = plt.figure()
        fig.subplots_adjust(right=0.75)
        ax = fig.add_subplot(111)

        '''Drawing multiple lines with different y axises in one graph'''
        lines = []
        labels = list(self.data[0][:-1])

        for num in xrange(len(self.data[0]) - 1):
            label = labels[num]
            if num == 0:
                dtype = t['{0}'.format(label)]
                line1, = ax.plot(pd.to_datetime(t.Time), dtype, color=self.uniqueish_color())
                lines.append(line1)
                ax.set_ylabel(label)
                ax.set_xlabel('Date')
            elif label != 'Time':
                dtype = t['{0}'.format(label)]
                par = ax.twinx()
                line2, = par.plot(pd.to_datetime(t.Time), dtype, color=self.uniqueish_color())
                lines.append(line2)
                par.set_ylabel(label)

        '''Formatting the date'''
        fig.autofmt_xdate()
        ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d')
        plt.title('Sensor Data Flow')

        '''Create the labels for different lines'''
        ax.legend(lines, labels, loc='lower left')
        plt.draw()
        plt.show()

def main():
    if len(sys.argv) != 2:
        print "Error with the parameters! Please specify the file path!"
        sys.exit(2)
    filename = sys.argv[1]

    gg = Gen_Graph(filename)

    data = gg.readData()
    print "Original Data: "
    for i in data:
        print i
    print '=============================================================='

    gg.genGraph_m()

    print "Finished Drawing!"

if __name__ == "__main__":
    main()


# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Mathieu Blondel <mathieu@mblondel.org>
#          Olivier Grisel <olivier.grisel@ensta.org>
#          Andreas Mueller <amueller@ais.uni-bonn.de>
#          Joel Nothman <joel.nothman@gmail.com>
#          Hamzeh Alsalhi <ha258@cornell.edu>
# License: BSD 3 clause

from collections import defaultdict
import itertools
import array
import warnings

import numpy as np
import scipy.sparse as sp

from ..base import BaseEstimator, TransformerMixin

from ..utils.fixes import np_version
from ..utils.fixes import sparse_min_max
from ..utils.fixes import astype
from ..utils.fixes import in1d
from ..utils import deprecated, column_or_1d
from ..utils.validation import check_array
from ..utils.validation import _num_samples
from ..utils.multiclass import unique_labels
from ..utils.multiclass import type_of_target

from ..externals import six

zip = six.moves.zip
map = six.moves.map

__all__ = [
    'label_binarize',
    'LabelBinarizer',
    'LabelEncoder',
    'MultiLabelBinarizer',
]

def _check_numpy_unicode_bug(labels):
    """Check that user is not subject to an old numpy bug

    Fixed in master before 1.7.0:

      https://github.com/numpy/numpy/pull/243

    """
    if np_version[:3] < (1, 7, 0) and labels.dtype.kind == 'U':
        raise RuntimeError("NumPy < 1.7.0 does not implement searchsorted"
                           " on unicode data correctly. Please upgrade"
                           " NumPy to use LabelEncoder with unicode inputs.")

class LabelEncoder(BaseEstimator, TransformerMixin):
    """Encode labels with value between 0 and n_classes-1.

    Read more in the :ref:`User Guide <preprocessing_targets>`.

    Attributes
    ----------
    classes_ : array of shape (n_class,)
        Holds the label for each class.

    Examples
    --------
    `LabelEncoder` can be used to normalize labels.

    >>> from sklearn import preprocessing
    >>> le = preprocessing.LabelEncoder()
    >>> le.fit([1, 2, 2, 6])
    LabelEncoder()
    >>> le.classes_
    array([1, 2, 6])
    >>> le.transform([1, 1, 2, 6]) #doctest: +ELLIPSIS
    array([0, 0, 1, 2]...)
    >>> le.inverse_transform([0, 0, 1, 2])
    array([1, 1, 2, 6])

    It can also be used to transform non-numerical labels (as long as they are
    hashable and comparable) to numerical labels.

    >>> le = preprocessing.LabelEncoder()
    >>> le.fit(["paris", "paris", "tokyo", "amsterdam"])
    LabelEncoder()
    >>> list(le.classes_)
    ['amsterdam', 'paris', 'tokyo']
    >>> le.transform(["tokyo", "tokyo", "paris"]) #doctest: +ELLIPSIS
    array([2, 2, 1]...)
    >>> list(le.inverse_transform([2, 2, 1]))
    ['tokyo', 'tokyo', 'paris']

    """

    def _check_fitted(self):
        if not hasattr(self, "classes_"):
            raise ValueError("LabelEncoder was not fitted yet.")

    def fit(self, y):
        """Fit label encoder

        Parameters
        ----------
        y : array-like of shape (n_samples,)
            Target values.

        Returns
        -------
        self : returns an instance of self.
        """
        y = column_or_1d(y, warn=True)
        _check_numpy_unicode_bug(y)
        self.classes_ = np.unique(y)
        return self

    def fit_transform(self, y):
        """Fit label encoder and return encoded labels

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        y = column_or_1d(y, warn=True)
        _check_numpy_unicode_bug(y)
        self.classes_, y = np.unique(y, return_inverse=True)
        return y

    def transform(self, y):
        """Transform labels to normalized encoding.

        Parameters
        ----------
        y : array-like of shape [n_samples]
            Target values.

        Returns
        -------
        y : array-like of shape [n_samples]
        """
        self._check_fitted()

        classes = np.unique(y)
        _check_numpy_unicode_bug(classes)
        if len(np.intersect1d(classes, self.classes_)) < len(classes):
            diff = np.setdiff1d(classes, self.classes_)
            raise ValueError("y contains new labels: %s" % str(diff))
        return np.searchsorted(self.classes_, y)

    def inverse_transform(self, y):
        """Transform labels back to original encoding.

        Parameters
        ----------
        y : numpy array of shape [n_samples]
            Target values.

        Returns
        -------
        y : numpy array of shape [n_samples]
        """
        self._check_fitted()

        y = np.asarray(y)
        return self.classes_[y]

class LabelBinarizer(BaseEstimator, TransformerMixin):
    """Binarize labels in a one-vs-all fashion

    Several regression and binary classification algorithms are
    available in the scikit. A simple way to extend these algorithms
    to the multi-class classification case is to use the so-called
    one-vs-all scheme.

    At learning time, this simply consists in learning one regressor
    or binary classifier per class. In doing so, one needs to convert
    multi-class labels to binary labels (belong or does not belong
    to the class). LabelBinarizer makes this process easy with the
    transform method.

    At prediction time, one assigns the class for which the corresponding
    model gave the greatest confidence. LabelBinarizer makes this easy
    with the inverse_transform method.

    Read more in the :ref:`User Guide <preprocessing_targets>`.

    Parameters
    ----------

    neg_label : int (default: 0)
        Value with which negative labels must be encoded.

    pos_label : int (default: 1)
        Value with which positive labels must be encoded.

    sparse_output : boolean (default: False)
        True if the returned array from transform is desired to be in sparse
        CSR format.

    Attributes
    ----------
    classes_ : array of shape [n_class]
        Holds the label for each class.

    y_type_ : str,
        Represents the type of the target data as evaluated by
        utils.multiclass.type_of_target. Possible type are 'continuous',
        'continuous-multioutput', 'binary', 'multiclass',
        'mutliclass-multioutput', 'multilabel-sequences',
        'multilabel-indicator', and 'unknown'.

    multilabel_ : boolean
        True if the transformer was fitted on a multilabel rather than a
        multiclass set of labels. The ``multilabel_`` attribute is deprecated
        and will be removed in 0.18

    sparse_input_ : boolean,
        True if the input data to transform is given as a sparse matrix, False
        otherwise.

    indicator_matrix_ : str
        'sparse' when the input data to tansform is a multilable-indicator and
        is sparse, None otherwise. The ``indicator_matrix_`` attribute is
        deprecated as of version 0.16 and will be removed in 0.18

    Examples
    --------
    >>> from sklearn import preprocessing
    >>> lb = preprocessing.LabelBinarizer()
    >>> lb.fit([1, 2, 6, 4, 2])
    LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)
    >>> lb.classes_
    array([1, 2, 4, 6])
    >>> lb.transform([1, 6])
    array([[1, 0, 0, 0],
           [0, 0, 0, 1]])

    Binary targets transform to a column vector

    >>> lb = preprocessing.LabelBinarizer()
    >>> lb.fit_transform(['yes', 'no', 'no', 'yes'])
    array([[1],
           [0],
           [0],
           [1]])

    Passing a 2D matrix for multilabel classification

    >>> import numpy as np
    >>> lb.fit(np.array([[0, 1, 1], [1, 0, 0]]))
    LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)
    >>> lb.classes_
    array([0, 1, 2])
    >>> lb.transform([0, 1, 2, 1])
    array([[1, 0, 0],
           [0, 1, 0],
           [0, 0, 1],
           [0, 1, 0]])

    See also
    --------
    label_binarize : function to perform the transform operation of
        LabelBinarizer with fixed classes.
    """

    def __init__(self, neg_label=0, pos_label=1, sparse_output=False):
        if neg_label >= pos_label:
            raise ValueError("neg_label={0} must be strictly less than "
                             "pos_label={1}.".format(neg_label, pos_label))

        if sparse_output and (pos_label == 0 or neg_label != 0):
            raise ValueError("Sparse binarization is only supported with non "
                             "zero pos_label and zero neg_label, got "
                             "pos_label={0} and neg_label={1}"
                             "".format(pos_label, neg_label))

        self.neg_label = neg_label
        self.pos_label = pos_label
        self.sparse_output = sparse_output

    @property
    @deprecated("Attribute ``indicator_matrix_`` is deprecated and will be "
                "removed in 0.17. Use ``y_type_ == 'multilabel-indicator'`` "
                "instead")
    def indicator_matrix_(self):
        return self.y_type_ == 'multilabel-indicator'

    @property
    @deprecated("Attribute ``multilabel_`` is deprecated and will be removed "
                "in 0.17. Use ``y_type_.startswith('multilabel')`` "
                "instead")
    def multilabel_(self):
        return self.y_type_.startswith('multilabel')

    def _check_fitted(self):
        if not hasattr(self, "classes_"):
            raise ValueError("LabelBinarizer was not fitted yet.")

    def fit(self, y):
        """Fit label binarizer

        Parameters
        ----------
        y : numpy array of shape (n_samples,) or (n_samples, n_classes)
            Target values. The 2-d matrix should only contain 0 and 1,
            represents multilabel classification.

        Returns
        -------
        self : returns an instance of self.
        """
        self.y_type_ = type_of_target(y)
        if 'multioutput' in self.y_type_:
            raise ValueError("Multioutput target data is not supported with "
                             "label binarization")
        if _num_samples(y) == 0:
            raise ValueError('y has 0 samples: %r' % y)

        self.sparse_input_ = sp.issparse(y)
        self.classes_ = unique_labels(y)
        return self

    def transform(self, y):
        """Transform multi-class labels to binary labels

        The output of transform is sometimes referred to by some authors as the
        1-of-K coding scheme.

        Parameters
        ----------
        y : numpy array or sparse matrix of shape (n_samples,) or
            (n_samples, n_classes) Target values. The 2-d matrix should only
            contain 0 and 1, represents multilabel classification. Sparse
            matrix can be CSR, CSC, COO, DOK, or LIL.

        Returns
        -------
        Y : numpy array or CSR matrix of shape [n_samples, n_classes]
            Shape will be [n_samples, 1] for binary problems.
        """
        self._check_fitted()

        y_is_multilabel = type_of_target(y).startswith('multilabel')
        if y_is_multilabel and not self.y_type_.startswith('multilabel'):
            raise ValueError("The object was not fitted with multilabel"
                             " input.")

        return label_binarize(y, self.classes_,
                              pos_label=self.pos_label,
                              neg_label=self.neg_label,
                              sparse_output=self.sparse_output)

    def inverse_transform(self, Y, threshold=None):
        """Transform binary labels back to multi-class labels

        Parameters
        ----------
        Y : numpy array or sparse matrix with shape [n_samples, n_classes]
            Target values. All sparse matrices are converted to CSR before
            inverse transformation.

        threshold : float or None
            Threshold used in the binary and multi-label cases.

            Use 0 when:
                - Y contains the output of decision_function (classifier)
            Use 0.5 when:
                - Y contains the output of predict_proba

            If None, the threshold is assumed to be half way between
            neg_label and pos_label.

        Returns
        -------
        y : numpy array or CSR matrix of shape [n_samples] Target values.

        Notes
        -----
        In the case when the binary labels are fractional
        (probabilistic), inverse_transform chooses the class with the
        greatest value. Typically, this allows to use the output of a
        linear model's decision_function method directly as the input
        of inverse_transform.
        """
        self._check_fitted()

        if threshold is None:
            threshold = (self.pos_label + self.neg_label) / 2.

        if self.y_type_ == "multiclass":
            y_inv = _inverse_binarize_multiclass(Y, self.classes_)
        else:
            y_inv = _inverse_binarize_thresholding(Y, self.y_type_,
                                                   self.classes_, threshold)

        if self.sparse_input_:
            y_inv = sp.csr_matrix(y_inv)
        elif sp.issparse(y_inv):
            y_inv = y_inv.toarray()

        return y_inv

def label_binarize(y, classes, neg_label=0, pos_label=1,
                   sparse_output=False, multilabel=None):
    """Binarize labels in a one-vs-all fashion

    Several regression and binary classification algorithms are
    available in the scikit. A simple way to extend these algorithms
    to the multi-class classification case is to use the so-called
    one-vs-all scheme.

    This function makes it possible to compute this transformation for a
    fixed set of class labels known ahead of time.

    Parameters
    ----------
    y : array-like
        Sequence of integer labels or multilabel data to encode.

    classes : array-like of shape [n_classes]
        Uniquely holds the label for each class.

    neg_label : int (default: 0)
        Value with which negative labels must be encoded.

    pos_label : int (default: 1)
        Value with which positive labels must be encoded.

    sparse_output : boolean (default: False),
        Set to true if output binary array is desired in CSR sparse format

    Returns
    -------
    Y : numpy array or CSR matrix of shape [n_samples, n_classes]
        Shape will be [n_samples, 1] for binary problems.

    Examples
    --------
    >>> from sklearn.preprocessing import label_binarize
    >>> label_binarize([1, 6], classes=[1, 2, 4, 6])
    array([[1, 0, 0, 0],
           [0, 0, 0, 1]])

    The class ordering is preserved:

    >>> label_binarize([1, 6], classes=[1, 6, 4, 2])
    array([[1, 0, 0, 0],
           [0, 1, 0, 0]])

    Binary targets transform to a column vector

    >>> label_binarize(['yes', 'no', 'no', 'yes'], classes=['no', 'yes'])
    array([[1],
           [0],
           [0],
           [1]])

    See also
    --------
    LabelBinarizer : class used to wrap the functionality of label_binarize and
        allow for fitting to classes independently of the transform operation
    """
    if not isinstance(y, list):
        # XXX Workaround that will be removed when list of list format is
        # dropped
        y = check_array(y, accept_sparse='csr', ensure_2d=False, dtype=None)
    else:
        if _num_samples(y) == 0:
            raise ValueError('y has 0 samples: %r' % y)
    if neg_label >= pos_label:
        raise ValueError("neg_label={0} must be strictly less than "
                         "pos_label={1}.".format(neg_label, pos_label))

    if (sparse_output and (pos_label == 0 or neg_label != 0)):
        raise ValueError("Sparse binarization is only supported with non "
                         "zero pos_label and zero neg_label, got "
                         "pos_label={0} and neg_label={1}"
                         "".format(pos_label, neg_label))

    if multilabel is not None:
        warnings.warn("The multilabel parameter is deprecated as of version "
                      "0.15 and will be removed in 0.17. The parameter is no "
                      "longer necessary because the value is automatically "
                      "inferred.", DeprecationWarning)

    # To account for pos_label == 0 in the dense case
    pos_switch = pos_label == 0
    if pos_switch:
        pos_label = -neg_label

    y_type = type_of_target(y)
    if 'multioutput' in y_type:
        raise ValueError("Multioutput target data is not supported with label "
                         "binarization")

    n_samples = y.shape[0] if sp.issparse(y) else len(y)
    n_classes = len(classes)
    classes = np.asarray(classes)

    if y_type == "binary":
        if len(classes) == 1:
            Y = np.zeros((len(y), 1), dtype=np.int)
            Y += neg_label
            return Y
        elif len(classes) >= 3:
            y_type = "multiclass"

    sorted_class = np.sort(classes)
    if (y_type == "multilabel-indicator" and classes.size != y.shape[1]):
        raise ValueError("classes {0} missmatch with the labels {1}"
                         "found in the data".format(classes, unique_labels(y)))

    if y_type in ("binary", "multiclass"):
        y = column_or_1d(y)

        # pick out the known labels from y
        y_in_classes = in1d(y, classes)
        y_seen = y[y_in_classes]
        indices = np.searchsorted(sorted_class, y_seen)
        indptr = np.hstack((0, np.cumsum(y_in_classes)))

        data = np.empty_like(indices)
        data.fill(pos_label)
        Y = sp.csr_matrix((data, indices, indptr),
                          shape=(n_samples, n_classes))

    elif y_type == "multilabel-indicator":
        Y = sp.csr_matrix(y)
        if pos_label != 1:
            data = np.empty_like(Y.data)
            data.fill(pos_label)
            Y.data = data

    elif y_type == "multilabel-sequences":
        Y = MultiLabelBinarizer(classes=classes,
                                sparse_output=sparse_output).fit_transform(y)

        if sp.issparse(Y):
            Y.data[:] = pos_label
        else:
            Y[Y == 1] = pos_label
        return Y

    if not sparse_output:
        Y = Y.toarray()
        Y = astype(Y, int, copy=False)

        if neg_label != 0:
            Y[Y == 0] = neg_label

        if pos_switch:
            Y[Y == pos_label] = 0
    else:
        Y.data = astype(Y.data, int, copy=False)

    # preserve label ordering
    if np.any(classes != sorted_class):
        indices = np.searchsorted(sorted_class, classes)
        Y = Y[:, indices]

    if y_type == "binary":
        if sparse_output:
            Y = Y.getcol(-1)
        else:
            Y = Y[:, -1].reshape((-1, 1))

    return Y

def _inverse_binarize_multiclass(y, classes):
    """Inverse label binarization transformation for multiclass.

    Multiclass uses the maximal score instead of a threshold.
    """
    classes = np.asarray(classes)

    if sp.issparse(y):
        # Find the argmax for each row in y where y is a CSR matrix

        y = y.tocsr()
        n_samples, n_outputs = y.shape
        outputs = np.arange(n_outputs)
        row_max = sparse_min_max(y, 1)[1]
        row_nnz = np.diff(y.indptr)

        y_data_repeated_max = np.repeat(row_max, row_nnz)
        # picks out all indices obtaining the maximum per row
        y_i_all_argmax = np.flatnonzero(y_data_repeated_max == y.data)

        # For corner case where last row has a max of 0
        if row_max[-1] == 0:
            y_i_all_argmax = np.append(y_i_all_argmax, [len(y.data)])

        # Gets the index of the first argmax in each row from y_i_all_argmax
        index_first_argmax = np.searchsorted(y_i_all_argmax, y.indptr[:-1])
        # first argmax of each row
        y_ind_ext = np.append(y.indices, [0])
        y_i_argmax = y_ind_ext[y_i_all_argmax[index_first_argmax]]
        # Handle rows of all 0
        y_i_argmax[np.where(row_nnz == 0)[0]] = 0

        # Handles rows with max of 0 that contain negative numbers
        samples = np.arange(n_samples)[(row_nnz > 0) &
                                       (row_max.ravel() == 0)]
        for i in samples:
            ind = y.indices[y.indptr[i]:y.indptr[i + 1]]
            y_i_argmax[i] = classes[np.setdiff1d(outputs, ind)][0]

        return classes[y_i_argmax]
    else:
        return classes.take(y.argmax(axis=1), mode="clip")

def _inverse_binarize_thresholding(y, output_type, classes, threshold):
    """Inverse label binarization transformation using thresholding."""

    if output_type == "binary" and y.ndim == 2 and y.shape[1] > 2:
        raise ValueError("output_type='binary', but y.shape = {0}".
                         format(y.shape))

    if output_type != "binary" and y.shape[1] != len(classes):
        raise ValueError("The number of class is not equal to the number of "
                         "dimension of y.")

    classes = np.asarray(classes)

    # Perform thresholding
    if sp.issparse(y):
        if threshold > 0:
            if y.format not in ('csr', 'csc'):
                y = y.tocsr()
            y.data = np.array(y.data > threshold, dtype=np.int)
            y.eliminate_zeros()
        else:
            y = np.array(y.toarray() > threshold, dtype=np.int)
    else:
        y = np.array(y > threshold, dtype=np.int)

    # Inverse transform data
    if output_type == "binary":
        if sp.issparse(y):
            y = y.toarray()
        if y.ndim == 2 and y.shape[1] == 2:
            return classes[y[:, 1]]
        else:
            if len(classes) == 1:
                y = np.empty(len(y), dtype=classes.dtype)
                y.fill(classes[0])
                return y
            else:
                return classes[y.ravel()]

    elif output_type == "multilabel-indicator":
        return y

    elif output_type == "multilabel-sequences":
        warnings.warn('Direct support for sequence of sequences multilabel '
                      'representation will be unavailable from version 0.17. '
                      'Use sklearn.preprocessing.MultiLabelBinarizer to '
                      'convert to a label indicator representation.',
                      DeprecationWarning)
        mlb = MultiLabelBinarizer(classes=classes).fit([])
        return mlb.inverse_transform(y)

    else:
        raise ValueError("{0} format is not supported".format(output_type))

class MultiLabelBinarizer(BaseEstimator, TransformerMixin):
    """Transform between iterable of iterables and a multilabel format

    Although a list of sets or tuples is a very intuitive format for multilabel
    data, it is unwieldy to process. This transformer converts between this
    intuitive format and the supported multilabel format: a (samples x classes)
    binary matrix indicating the presence of a class label.

    Parameters
    ----------
    classes : array-like of shape [n_classes] (optional)
        Indicates an ordering for the class labels

    sparse_output : boolean (default: False),
        Set to true if output binary array is desired in CSR sparse format

    Attributes
    ----------
    classes_ : array of labels
        A copy of the `classes` parameter where provided,
        or otherwise, the sorted set of classes found when fitting.

    Examples
    --------
    >>> mlb = MultiLabelBinarizer()
    >>> mlb.fit_transform([(1, 2), (3,)])
    array([[1, 1, 0],
           [0, 0, 1]])
    >>> mlb.classes_
    array([1, 2, 3])

    >>> mlb.fit_transform([set(['sci-fi', 'thriller']), set(['comedy'])])
    array([[0, 1, 1],
           [1, 0, 0]])
    >>> list(mlb.classes_)
    ['comedy', 'sci-fi', 'thriller']

    """
    def __init__(self, classes=None, sparse_output=False):
        self.classes = classes
        self.sparse_output = sparse_output

    def fit(self, y):
        """Fit the label sets binarizer, storing `classes_`

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        self : returns this MultiLabelBinarizer instance
        """
        if self.classes is None:
            classes = sorted(set(itertools.chain.from_iterable(y)))
        else:
            classes = self.classes
        dtype = np.int if all(isinstance(c, int) for c in classes) else object
        self.classes_ = np.empty(len(classes), dtype=dtype)
        self.classes_[:] = classes
        return self

    def fit_transform(self, y):
        """Fit the label sets binarizer and transform the given label sets

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        y_indicator : array or CSR matrix, shape (n_samples, n_classes)
            A matrix such that `y_indicator[i, j] = 1` iff `classes_[j]` is in
            `y[i]`, and 0 otherwise.
        """
        if self.classes is not None:
            return self.fit(y).transform(y)

        # Automatically increment on new class
        class_mapping = defaultdict(int)
        class_mapping.default_factory = class_mapping.__len__
        yt = self._transform(y, class_mapping)

        # sort classes and reorder columns
        tmp = sorted(class_mapping, key=class_mapping.get)

        # (make safe for tuples)
        dtype = np.int if all(isinstance(c, int) for c in tmp) else object
        class_mapping = np.empty(len(tmp), dtype=dtype)
        class_mapping[:] = tmp
        self.classes_, inverse = np.unique(class_mapping, return_inverse=True)
        yt.indices = np.take(inverse, yt.indices)

        if not self.sparse_output:
            yt = yt.toarray()

        return yt

    def transform(self, y):
        """Transform the given label sets

        Parameters
        ----------
        y : iterable of iterables
            A set of labels (any orderable and hashable object) for each
            sample. If the `classes` parameter is set, `y` will not be
            iterated.

        Returns
        -------
        y_indicator : array or CSR matrix, shape (n_samples, n_classes)
            A matrix such that `y_indicator[i, j] = 1` iff `classes_[j]` is in
            `y[i]`, and 0 otherwise.
        """
        class_to_index = dict(zip(self.classes_, range(len(self.classes_))))
        yt = self._transform(y, class_to_index)

        if not self.sparse_output:
            yt = yt.toarray()

        return yt

    def _transform(self, y, class_mapping):
        """Transforms the label sets with a given mapping

        Parameters
        ----------
        y : iterable of iterables
        class_mapping : Mapping
            Maps from label to column index in label indicator matrix

        Returns
        -------
        y_indicator : sparse CSR matrix, shape (n_samples, n_classes)
            Label indicator matrix
        """
        indices = array.array('i')
        indptr = array.array('i', [0])
        for labels in y:
            indices.extend(set(class_mapping[label] for label in labels))
            indptr.append(len(indices))
        data = np.ones(len(indices), dtype=int)

        return sp.csr_matrix((data, indices, indptr),
                             shape=(len(indptr) - 1, len(class_mapping)))

    def inverse_transform(self, yt):
        """Transform the given indicator matrix into label sets

        Parameters
        ----------
        yt : array or sparse matrix of shape (n_samples, n_classes)
            A matrix containing only 1s ands 0s.

        Returns
        -------
        y : list of tuples
            The set of labels for each sample such that `y[i]` consists of
            `classes_[j]` for each `yt[i, j] == 1`.
        """
        if yt.shape[1] != len(self.classes_):
            raise ValueError('Expected indicator for {0} classes, but got {1}'
                             .format(len(self.classes_), yt.shape[1]))

        if sp.issparse(yt):
            yt = yt.tocsr()
            if len(yt.data) != 0 and len(np.setdiff1d(yt.data, [0, 1])) > 0:
                raise ValueError('Expected only 0s and 1s in label indicator.')
            return [tuple(self.classes_.take(yt.indices[start:end]))
                    for start, end in zip(yt.indptr[:-1], yt.indptr[1:])]
        else:
            unexpected = np.setdiff1d(yt, [0, 1])
            if len(unexpected) > 0:
                raise ValueError('Expected only 0s and 1s in label indicator. '
                                 'Also got {0}'.format(unexpected))
            return [tuple(self.classes_.compress(indicators)) for indicators
                    in yt]


import numpy as np

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises

from sklearn import datasets
from sklearn.decomposition import PCA
from sklearn.decomposition import RandomizedPCA
from sklearn.decomposition.pca import _assess_dimension_
from sklearn.decomposition.pca import _infer_dimension_

iris = datasets.load_iris()

def test_pca():
    """PCA on dense arrays"""
    pca = PCA(n_components=2)
    X = iris.data
    X_r = pca.fit(X).transform(X)
    np.testing.assert_equal(X_r.shape[1], 2)

    X_r2 = pca.fit_transform(X)
    assert_array_almost_equal(X_r, X_r2)

    pca = PCA()
    pca.fit(X)
    assert_almost_equal(pca.explained_variance_ratio_.sum(), 1.0, 3)

    X_r = pca.transform(X)
    X_r2 = pca.fit_transform(X)

    assert_array_almost_equal(X_r, X_r2)

    # Test get_covariance and get_precision with n_components == n_features
    # with n_components < n_features and with n_components == 0
    for n_components in [0, 2, X.shape[1]]:
        pca.n_components = n_components
        pca.fit(X)
        cov = pca.get_covariance()
        precision = pca.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]), 12)

def test_whitening():
    """Check that PCA output has unit-variance"""
    rng = np.random.RandomState(0)
    n_samples = 100
    n_features = 80
    n_components = 30
    rank = 50

    # some low rank data with correlated features
    X = np.dot(rng.randn(n_samples, rank),
               np.dot(np.diag(np.linspace(10.0, 1.0, rank)),
                      rng.randn(rank, n_features)))
    # the component-wise variance of the first 50 features is 3 times the
    # mean component-wise variance of the remaingin 30 features
    X[:, :50] *= 3

    assert_equal(X.shape, (n_samples, n_features))

    # the component-wise variance is thus highly varying:
    assert_almost_equal(X.std(axis=0).std(), 43.9, 1)

    for this_PCA, copy in [(x, y) for x in (PCA, RandomizedPCA)
                           for y in (True, False)]:
        # whiten the data while projecting to the lower dim subspace
        X_ = X.copy()  # make sure we keep an original across iterations.
        pca = this_PCA(n_components=n_components, whiten=True, copy=copy)
        # test fit_transform
        X_whitened = pca.fit_transform(X_.copy())
        assert_equal(X_whitened.shape, (n_samples, n_components))
        X_whitened2 = pca.transform(X_)
        assert_array_almost_equal(X_whitened, X_whitened2)

        assert_almost_equal(X_whitened.std(axis=0), np.ones(n_components))
        assert_almost_equal(X_whitened.mean(axis=0), np.zeros(n_components))

        X_ = X.copy()
        pca = this_PCA(n_components=n_components, whiten=False,
                       copy=copy).fit(X_)
        X_unwhitened = pca.transform(X_)
        assert_equal(X_unwhitened.shape, (n_samples, n_components))

        # in that case the output components still have varying variances
        assert_almost_equal(X_unwhitened.std(axis=0).std(), 74.1, 1)
        # we always center, so no test for non-centering.

def test_explained_variance():
    """Check that PCA output has unit-variance"""
    rng = np.random.RandomState(0)
    n_samples = 100
    n_features = 80

    X = rng.randn(n_samples, n_features)

    pca = PCA(n_components=2).fit(X)
    rpca = RandomizedPCA(n_components=2, random_state=42).fit(X)
    assert_array_almost_equal(pca.explained_variance_,
                              rpca.explained_variance_, 1)
    assert_array_almost_equal(pca.explained_variance_ratio_,
                              rpca.explained_variance_ratio_, 3)

    # compare to empirical variances
    X_pca = pca.transform(X)
    assert_array_almost_equal(pca.explained_variance_,
                              np.var(X_pca, axis=0))

    X_rpca = rpca.transform(X)
    assert_array_almost_equal(rpca.explained_variance_,
                              np.var(X_rpca, axis=0))

def test_pca_check_projection():
    """Test that the projection of data is correct"""
    rng = np.random.RandomState(0)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    Yt = PCA(n_components=2).fit(X).transform(Xt)
    Yt /= np.sqrt((Yt ** 2).sum())

    assert_almost_equal(np.abs(Yt[0][0]), 1., 1)

def test_pca_inverse():
    """Test that the projection of data can be inverted"""
    rng = np.random.RandomState(0)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed
    # signal (since the data is almost of rank n_components)
    pca = PCA(n_components=2).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=3)

    # same as above with whitening (approximate reconstruction)
    pca = PCA(n_components=2, whiten=True)
    pca.fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=3)

def test_pca_validation():
    X = [[0, 1], [1, 0]]
    for n_components in [-1, 3]:
        assert_raises(ValueError, PCA(n_components).fit, X)

def test_randomized_pca_check_projection():
    """Test that the projection by RandomizedPCA on dense data is correct"""
    rng = np.random.RandomState(0)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    Yt = RandomizedPCA(n_components=2, random_state=0).fit(X).transform(Xt)
    Yt /= np.sqrt((Yt ** 2).sum())

    assert_almost_equal(np.abs(Yt[0][0]), 1., 1)

def test_randomized_pca_check_list():
    """Test that the projection by RandomizedPCA on list data is correct"""
    X = [[1.0, 0.0], [0.0, 1.0]]
    X_transformed = RandomizedPCA(n_components=1,
                                  random_state=0).fit(X).transform(X)
    assert_equal(X_transformed.shape, (2, 1))
    assert_almost_equal(X_transformed.mean(), 0.00, 2)
    assert_almost_equal(X_transformed.std(), 0.71, 2)

def test_randomized_pca_inverse():
    """Test that RandomizedPCA is inversible on dense data"""
    rng = np.random.RandomState(0)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed signal
    # (since the data is almost of rank n_components)
    pca = RandomizedPCA(n_components=2, random_state=0).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=2)

    # same as above with whitening (approximate reconstruction)
    pca = RandomizedPCA(n_components=2, whiten=True,
                        random_state=0).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    relative_max_delta = (np.abs(X - Y_inverse) / np.abs(X).mean()).max()
    assert_almost_equal(relative_max_delta, 0.11, decimal=2)

def test_pca_dim():
    """Check automated dimensionality setting"""
    rng = np.random.RandomState(0)
    n, p = 100, 5
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    pca = PCA(n_components='mle').fit(X)
    assert_equal(pca.n_components, 'mle')
    assert_equal(pca.n_components_, 1)

def test_infer_dim_1():
    """TODO: explain what this is testing

    Or at least use explicit variable names...
    """
    n, p = 1000, 5
    rng = np.random.RandomState(0)
    X = (rng.randn(n, p) * .1 + rng.randn(n, 1) * np.array([3, 4, 5, 1, 2])
         + np.array([1, 0, 7, 4, 6]))
    pca = PCA(n_components=p)
    pca.fit(X)
    spect = pca.explained_variance_
    ll = []
    for k in range(p):
        ll.append(_assess_dimension_(spect, k, n, p))
    ll = np.array(ll)
    assert_greater(ll[1], ll.max() - .01 * n)

def test_infer_dim_2():
    """TODO: explain what this is testing

    Or at least use explicit variable names...
    """
    n, p = 1000, 5
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    X[10:20] += np.array([6, 0, 7, 2, -1])
    pca = PCA(n_components=p)
    pca.fit(X)
    spect = pca.explained_variance_
    assert_greater(_infer_dimension_(spect, n, p), 1)

def test_infer_dim_3():
    """
    """
    n, p = 100, 5
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    X[10:20] += np.array([6, 0, 7, 2, -1])
    X[30:40] += 2 * np.array([-1, 1, -1, 1, -1])
    pca = PCA(n_components=p)
    pca.fit(X)
    spect = pca.explained_variance_
    assert_greater(_infer_dimension_(spect, n, p), 2)

def test_infer_dim_by_explained_variance():
    X = iris.data
    pca = PCA(n_components=0.95)
    pca.fit(X)
    assert_equal(pca.n_components, 0.95)
    assert_equal(pca.n_components_, 2)

    pca = PCA(n_components=0.01)
    pca.fit(X)
    assert_equal(pca.n_components, 0.01)
    assert_equal(pca.n_components_, 1)

    rng = np.random.RandomState(0)
    # more features than samples
    X = rng.rand(5, 20)
    pca = PCA(n_components=.5).fit(X)
    assert_equal(pca.n_components, 0.5)
    assert_equal(pca.n_components_, 2)

def test_pca_score():
    """Test that probabilistic PCA scoring yields a reasonable score"""
    n, p = 1000, 3
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1 + np.array([3, 4, 5])
    pca = PCA(n_components=2)
    pca.fit(X)
    ll1 = pca.score(X)
    h = -0.5 * np.log(2 * np.pi * np.exp(1) * 0.1 ** 2) * p
    np.testing.assert_almost_equal(ll1 / h, 1, 0)

def test_pca_score2():
    """Test that probabilistic PCA correctly separated different datasets"""
    n, p = 100, 3
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1 + np.array([3, 4, 5])
    pca = PCA(n_components=2)
    pca.fit(X)
    ll1 = pca.score(X)
    ll2 = pca.score(rng.randn(n, p) * .2 + np.array([3, 4, 5]))
    assert_greater(ll1, ll2)

    # Test that it gives the same scores if whiten=True
    pca = PCA(n_components=2, whiten=True)
    pca.fit(X)
    ll2 = pca.score(X)
    assert_almost_equal(ll1, ll2)

def test_pca_score3():
    """Check that probabilistic PCA selects the right model"""
    n, p = 200, 3
    rng = np.random.RandomState(0)
    Xl = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5])
          + np.array([1, 0, 7]))
    Xt = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5])
          + np.array([1, 0, 7]))
    ll = np.zeros(p)
    for k in range(p):
        pca = PCA(n_components=k)
        pca.fit(Xl)
        ll[k] = pca.score(Xt)

    assert_true(ll.argmax() == 1)

if __name__ == '__main__':
    import nose
    nose.run(argv=['', __file__])