import numpy as np

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_raises

from sklearn import datasets
from sklearn.decomposition import PCA
from sklearn.decomposition import RandomizedPCA
from sklearn.decomposition.pca import _assess_dimension_
from sklearn.decomposition.pca import _infer_dimension_

iris = datasets.load_iris()

def test_pca():
    # PCA on dense arrays
    pca = PCA(n_components=2)
    X = iris.data
    X_r = pca.fit(X).transform(X)
    np.testing.assert_equal(X_r.shape[1], 2)

    X_r2 = pca.fit_transform(X)
    assert_array_almost_equal(X_r, X_r2)

    pca = PCA()
    pca.fit(X)
    assert_almost_equal(pca.explained_variance_ratio_.sum(), 1.0, 3)

    X_r = pca.transform(X)
    X_r2 = pca.fit_transform(X)

    assert_array_almost_equal(X_r, X_r2)

    # Test get_covariance and get_precision with n_components == n_features
    # with n_components < n_features and with n_components == 0
    for n_components in [0, 2, X.shape[1]]:
        pca.n_components = n_components
        pca.fit(X)
        cov = pca.get_covariance()
        precision = pca.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]), 12)

def test_whitening():
    # Check that PCA output has unit-variance
    rng = np.random.RandomState(0)
    n_samples = 100
    n_features = 80
    n_components = 30
    rank = 50

    # some low rank data with correlated features
    X = np.dot(rng.randn(n_samples, rank),
               np.dot(np.diag(np.linspace(10.0, 1.0, rank)),
                      rng.randn(rank, n_features)))
    # the component-wise variance of the first 50 features is 3 times the
    # mean component-wise variance of the remaingin 30 features
    X[:, :50] *= 3

    assert_equal(X.shape, (n_samples, n_features))

    # the component-wise variance is thus highly varying:
    assert_almost_equal(X.std(axis=0).std(), 43.9, 1)

    for this_PCA, copy in [(x, y) for x in (PCA, RandomizedPCA)
                           for y in (True, False)]:
        # whiten the data while projecting to the lower dim subspace
        X_ = X.copy()  # make sure we keep an original across iterations.
        pca = this_PCA(n_components=n_components, whiten=True, copy=copy)
        # test fit_transform
        X_whitened = pca.fit_transform(X_.copy())
        assert_equal(X_whitened.shape, (n_samples, n_components))
        X_whitened2 = pca.transform(X_)
        assert_array_almost_equal(X_whitened, X_whitened2)

        assert_almost_equal(X_whitened.std(axis=0), np.ones(n_components))
        assert_almost_equal(X_whitened.mean(axis=0), np.zeros(n_components))

        X_ = X.copy()
        pca = this_PCA(n_components=n_components, whiten=False,
                       copy=copy).fit(X_)
        X_unwhitened = pca.transform(X_)
        assert_equal(X_unwhitened.shape, (n_samples, n_components))

        # in that case the output components still have varying variances
        assert_almost_equal(X_unwhitened.std(axis=0).std(), 74.1, 1)
        # we always center, so no test for non-centering.

def test_explained_variance():
    # Check that PCA output has unit-variance
    rng = np.random.RandomState(0)
    n_samples = 100
    n_features = 80

    X = rng.randn(n_samples, n_features)

    pca = PCA(n_components=2).fit(X)
    rpca = RandomizedPCA(n_components=2, random_state=42).fit(X)
    assert_array_almost_equal(pca.explained_variance_,
                              rpca.explained_variance_, 1)
    assert_array_almost_equal(pca.explained_variance_ratio_,
                              rpca.explained_variance_ratio_, 3)

    # compare to empirical variances
    X_pca = pca.transform(X)
    assert_array_almost_equal(pca.explained_variance_,
                              np.var(X_pca, axis=0))

    X_rpca = rpca.transform(X)
    assert_array_almost_equal(rpca.explained_variance_,
                              np.var(X_rpca, axis=0))

def test_pca_check_projection():
    # Test that the projection of data is correct
    rng = np.random.RandomState(0)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    Yt = PCA(n_components=2).fit(X).transform(Xt)
    Yt /= np.sqrt((Yt ** 2).sum())

    assert_almost_equal(np.abs(Yt[0][0]), 1., 1)

def test_pca_inverse():
    # Test that the projection of data can be inverted
    rng = np.random.RandomState(0)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed
    # signal (since the data is almost of rank n_components)
    pca = PCA(n_components=2).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=3)

    # same as above with whitening (approximate reconstruction)
    pca = PCA(n_components=2, whiten=True)
    pca.fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=3)

def test_pca_validation():
    X = [[0, 1], [1, 0]]
    for n_components in [-1, 3]:
        assert_raises(ValueError, PCA(n_components).fit, X)

def test_randomized_pca_check_projection():
    # Test that the projection by RandomizedPCA on dense data is correct
    rng = np.random.RandomState(0)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    Yt = RandomizedPCA(n_components=2, random_state=0).fit(X).transform(Xt)
    Yt /= np.sqrt((Yt ** 2).sum())

    assert_almost_equal(np.abs(Yt[0][0]), 1., 1)

def test_randomized_pca_check_list():
    # Test that the projection by RandomizedPCA on list data is correct
    X = [[1.0, 0.0], [0.0, 1.0]]
    X_transformed = RandomizedPCA(n_components=1,
                                  random_state=0).fit(X).transform(X)
    assert_equal(X_transformed.shape, (2, 1))
    assert_almost_equal(X_transformed.mean(), 0.00, 2)
    assert_almost_equal(X_transformed.std(), 0.71, 2)

def test_randomized_pca_inverse():
    # Test that RandomizedPCA is inversible on dense data
    rng = np.random.RandomState(0)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed signal
    # (since the data is almost of rank n_components)
    pca = RandomizedPCA(n_components=2, random_state=0).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=2)

    # same as above with whitening (approximate reconstruction)
    pca = RandomizedPCA(n_components=2, whiten=True,
                        random_state=0).fit(X)
    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    relative_max_delta = (np.abs(X - Y_inverse) / np.abs(X).mean()).max()
    assert_almost_equal(relative_max_delta, 0.11, decimal=2)

def test_pca_dim():
    # Check automated dimensionality setting
    rng = np.random.RandomState(0)
    n, p = 100, 5
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    pca = PCA(n_components='mle').fit(X)
    assert_equal(pca.n_components, 'mle')
    assert_equal(pca.n_components_, 1)

def test_infer_dim_1():
    # TODO: explain what this is testing
    # Or at least use explicit variable names...
    n, p = 1000, 5
    rng = np.random.RandomState(0)
    X = (rng.randn(n, p) * .1 + rng.randn(n, 1) * np.array([3, 4, 5, 1, 2])
         + np.array([1, 0, 7, 4, 6]))
    pca = PCA(n_components=p)
    pca.fit(X)
    spect = pca.explained_variance_
    ll = []
    for k in range(p):
        ll.append(_assess_dimension_(spect, k, n, p))
    ll = np.array(ll)
    assert_greater(ll[1], ll.max() - .01 * n)

def test_infer_dim_2():
    # TODO: explain what this is testing
    # Or at least use explicit variable names...
    n, p = 1000, 5
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    X[10:20] += np.array([6, 0, 7, 2, -1])
    pca = PCA(n_components=p)
    pca.fit(X)
    spect = pca.explained_variance_
    assert_greater(_infer_dimension_(spect, n, p), 1)

def test_infer_dim_3():
    n, p = 100, 5
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5, 1, 2])
    X[10:20] += np.array([6, 0, 7, 2, -1])
    X[30:40] += 2 * np.array([-1, 1, -1, 1, -1])
    pca = PCA(n_components=p)
    pca.fit(X)
    spect = pca.explained_variance_
    assert_greater(_infer_dimension_(spect, n, p), 2)

def test_infer_dim_by_explained_variance():
    X = iris.data
    pca = PCA(n_components=0.95)
    pca.fit(X)
    assert_equal(pca.n_components, 0.95)
    assert_equal(pca.n_components_, 2)

    pca = PCA(n_components=0.01)
    pca.fit(X)
    assert_equal(pca.n_components, 0.01)
    assert_equal(pca.n_components_, 1)

    rng = np.random.RandomState(0)
    # more features than samples
    X = rng.rand(5, 20)
    pca = PCA(n_components=.5).fit(X)
    assert_equal(pca.n_components, 0.5)
    assert_equal(pca.n_components_, 2)

def test_pca_score():
    # Test that probabilistic PCA scoring yields a reasonable score
    n, p = 1000, 3
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1 + np.array([3, 4, 5])
    pca = PCA(n_components=2)
    pca.fit(X)
    ll1 = pca.score(X)
    h = -0.5 * np.log(2 * np.pi * np.exp(1) * 0.1 ** 2) * p
    np.testing.assert_almost_equal(ll1 / h, 1, 0)

def test_pca_score2():
    # Test that probabilistic PCA correctly separated different datasets
    n, p = 100, 3
    rng = np.random.RandomState(0)
    X = rng.randn(n, p) * .1 + np.array([3, 4, 5])
    pca = PCA(n_components=2)
    pca.fit(X)
    ll1 = pca.score(X)
    ll2 = pca.score(rng.randn(n, p) * .2 + np.array([3, 4, 5]))
    assert_greater(ll1, ll2)

    # Test that it gives the same scores if whiten=True
    pca = PCA(n_components=2, whiten=True)
    pca.fit(X)
    ll2 = pca.score(X)
    assert_almost_equal(ll1, ll2)

def test_pca_score3():
    # Check that probabilistic PCA selects the right model
    n, p = 200, 3
    rng = np.random.RandomState(0)
    Xl = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5])
          + np.array([1, 0, 7]))
    Xt = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5])
          + np.array([1, 0, 7]))
    ll = np.zeros(p)
    for k in range(p):
        pca = PCA(n_components=k)
        pca.fit(Xl)
        ll[k] = pca.score(Xt)

    assert_true(ll.argmax() == 1)


"""
The :mod:`sklearn.preprocessing` module includes scaling, centering,
normalization, binarization and imputation methods.
"""

from ._function_transformer import FunctionTransformer

from .data import Binarizer
from .data import KernelCenterer
from .data import MinMaxScaler
from .data import MaxAbsScaler
from .data import Normalizer
from .data import RobustScaler
from .data import StandardScaler
from .data import add_dummy_feature
from .data import binarize
from .data import normalize
from .data import scale
from .data import robust_scale
from .data import maxabs_scale
from .data import minmax_scale
from .data import OneHotEncoder

from .data import PolynomialFeatures

from .label import label_binarize
from .label import LabelBinarizer
from .label import LabelEncoder
from .label import MultiLabelBinarizer

from .imputation import Imputer

__all__ = [
    'Binarizer',
    'FunctionTransformer',
    'Imputer',
    'KernelCenterer',
    'LabelBinarizer',
    'LabelEncoder',
    'MultiLabelBinarizer',
    'MinMaxScaler',
    'MaxAbsScaler',
    'Normalizer',
    'OneHotEncoder',
    'RobustScaler',
    'StandardScaler',
    'add_dummy_feature',
    'PolynomialFeatures',
    'binarize',
    'normalize',
    'scale',
    'robust_scale',
    'maxabs_scale',
    'minmax_scale',
    'label_binarize',
]


# -*- coding: utf-8 -*-
r"""
============================
Parser for Jupyter notebooks
============================

Class that holds the Ipython notebook information

"""
# Author: Óscar Nájera
# License: 3-clause BSD

from __future__ import division, absolute_import, print_function
import json
import os
import re
import sys

def ipy_notebook_skeleton():
    """Returns a dictionary with the elements of a Jupyter notebook"""
    py_version = sys.version_info
    notebook_skeleton = {
        "cells": [],
        "metadata": {
            "kernelspec": {
                "display_name": "Python " + str(py_version[0]),
                "language": "python",
                "name": "python" + str(py_version[0])
            },
            "language_info": {
                "codemirror_mode": {
                    "name": "ipython",
                    "version": py_version[0]
                },
                "file_extension": ".py",
                "mimetype": "text/x-python",
                "name": "python",
                "nbconvert_exporter": "python",
                "pygments_lexer": "ipython" + str(py_version[0]),
                "version": '{0}.{1}.{2}'.format(*sys.version_info[:3])
            }
        },
        "nbformat": 4,
        "nbformat_minor": 0
    }
    return notebook_skeleton

def rst2md(text):
    """Converts the RST text from the examples docstrigs and comments
    into markdown text for the IPython notebooks"""

    top_heading = re.compile(r'^=+$\s^([\w\s-]+)^=+$', flags=re.M)
    text = re.sub(top_heading, r'# \1', text)

    math_eq = re.compile(r'^\.\. math::((?:.+)?(?:\n+^  .+)*)', flags=re.M)
    text = re.sub(math_eq,
                  lambda match: r'$${0}$$'.format(match.group(1).strip()),
                  text)
    inline_math = re.compile(r':math:`(.+)`')
    text = re.sub(inline_math, r'$\1$', text)

    return text

class Notebook(object):
    """Ipython notebook object

    Constructs the file cell-by-cell and writes it at the end"""

    def __init__(self, file_name, target_dir):
        """Declare the skeleton of the notebook

        Parameters
        ----------
        file_name : str
            original script file name, .py extension will be renamed
        target_dir: str
            directory where notebook file is to be saved
        """

        self.file_name = file_name.replace('.py', '.ipynb')
        self.write_file = os.path.join(target_dir, self.file_name)
        self.work_notebook = ipy_notebook_skeleton()
        self.add_code_cell("%matplotlib inline")

    def add_code_cell(self, code):
        """Add a code cell to the notebook

        Parameters
        ----------
        code : str
            Cell content
        """

        code_cell = {
            "cell_type": "code",
            "execution_count": None,
            "metadata": {"collapsed": False},
            "outputs": [],
            "source": [code.strip()]
            }
        self.work_notebook["cells"].append(code_cell)

    def add_markdown_cell(self, text):
        """Add a markdown cell to the notebook

        Parameters
        ----------
        code : str
            Cell content
        """
        markdown_cell = {
            "cell_type": "markdown",
            "metadata": {},
            "source": [rst2md(text)]
        }
        self.work_notebook["cells"].append(markdown_cell)

    def save_file(self):
        """Saves the notebook to a file"""
        with open(self.write_file, 'w') as out_nb:
            json.dump(self.work_notebook, out_nb, indent=2)


__author__ = 'sonal'

import numpy as np
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
from matplotlib import cm
from mpl_toolkits.axes_grid1 import make_axes_locatable
import os
from math import *

def polar_plot(grid, title):

    #z = grid[:,8]
    u = grid[:,8]
    v = grid[:,9]
    plt.figure(figsize=(18,18))
    ax = plt.gca()
    #m = Basemap(projection='npaeqd',boundinglat=20,lon_0=-100.,resolution='l')
    m = Basemap(width=8000000, height=8000000, resolution='l', projection='lcc',\
             lat_0=60,lon_0=-100.)
    m.drawcoastlines()
    m.drawparallels(np.arange(-80.,81,20.),labels=[1,0,0,0],fontsize=10)
    m.drawmeridians(np.arange(-180.,181.,20.),labels=[0,0,0,1],fontsize=10)

    x,y =m(grid[:,7],grid[:,6])
    sc = m.scatter(x,y,s=abs(u),c=u,marker=',',cmap=cm.jet,alpha=0.9,edgecolors='none')
    plt.title(title)

    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    cb1 = plt.colorbar(sc,cax=cax)
    cb1.set_label("mA/m",fontsize=18)
    plt.savefig('GOCE_asc_EICSu_krigged_201104.png',bbox_inches='tight',pad_inches=0.2)
    #plt.show()

def asc_desc(data):
    asc = []
    desc = []
    lat = data[:,6]
    for i in range(0,len(data)-1):
        if lat[i+1] >= lat[i]:
            asc.append(i)
        else:
            desc.append(i)

    return asc, desc

SECS_data = np.loadtxt('EICS_201103_krigged.txt')

asc_idx, desc_idx= asc_desc(SECS_data)

asc_track = SECS_data[asc_idx,:]
desc_track = SECS_data[desc_idx,:]

polar_plot(asc_track,'GOCE Ascending EICS u component Krigged April, 2011')


"""This module contains functions for Zernike calculations. Mainly the private
function _zgen, a generator function for Zernike polynomials. The public
functions make use of _zgen to create height or slope maps in a unit
pupil, corresponding to individual Zernike terms.

Author: Ravi S. Jonnal / Werner Lab, UC Davis

Revision: 2.0 / 28 June 2014

"""

import numpy as np
from matplotlib import pyplot as plt
import sys
from time import sleep
import os

USE_CACHE_FILE = False

def fact(num):
    """Implementation of factorial function.
    """
    # Check that the number is an integer.
    assert(num%1==0)
    # Check that $num\geq 0$.
    assert(num>=0)

    # Compute $num!$ recursively.
    if num==0 or num==1:
        return 1
    else:
        return num * fact(num-1)

def choose(a,b):
    """Binomial coefficient, implemented using
    this module's factorial function.
    See [here](http://www.encyclopediaofmath.org/index.php/Newton_binomial) for detail.
    """
    assert(a>=b)
    return fact(a)/(fact(b)*fact(a-b))

def splitEquation(eqStr,width,bookend):
    if len(eqStr)<=width or len(eqStr)==0:
        return eqStr
    else:
        spaceIndices = []
        idx = 0
        while idx>-1:
            idx = eqStr.find(' ',idx+1)
            spaceIndices.append(idx)
        spaceIndices = spaceIndices[:-1]
        idxList = [x for x in spaceIndices if x<width]
        if len(idxList)==0:
            return eqStr
        else:
            idx = idxList[-1]
            head = eqStr[:idx]
            innards = ' ' + bookend + '\n' + bookend
            tail = splitEquation(eqStr[idx:],width,bookend)
            test =head + innards + tail
            return test

class Zernike:

    def __init__(self):

        if USE_CACHE_FILE:
            cachedir = './cache/'
            self._cachefn = os.path.join(cachedir,'zernike_cache.txt')

            if not os.path.exists(cachedir):
                os.makedirs(cachedir)

            try:
                self._termMatrix = np.loadtxt(self._cachefn).astype(np.int32)
            except Exception as e:
                print 'No term cache file. Creating.'
                self._termMatrix = np.array([])
                np.savetxt(self._cachefn,self._termMatrix)

        # Make a dictionary of precomputed coefficients, using the cache file.
        # This dictionary will be used to look up values when they exist in
        # the dictionary, and will recompute them otherwise.
        self._termDict = {}

        if USE_CACHE_FILE:
            for row in self._termMatrix:
                n,m,kindIndex,s,j,k = row[:6]
                t1,t2,t3,c,tXexp,tYexp = row[6:]
                self._termDict[(n,m,kindIndex,s,j,k)] = (t1,t2,t3,c,tXexp,tYexp)

        # The functions in this class can be asked for phase height,
        # or partial x or partial y derivatives. 'Kind' refers to
        # which of these is requested. Numerical encodings for 'kind'
        # permit some arithmetical simplicity and generality
        # (utilizing a number associated with the kind in a single
        # equation, rather than having different sets of equations
        # for each kind case).
        self._kindDictionary = {}
        self._kindDictionary['h'] = 0
        self._kindDictionary['dx'] = 1
        self._kindDictionary['dy'] = 2

    def j2nm(self,j):
        n = np.ceil((-3+np.sqrt(9+8*j))/2)
        m = 2*j-n*(n+2)
        return np.int(n),np.int(m)

    def nm2j(self,n,m):
        return np.int(n*(n+1)/2.0+(n+m)/2.0)

    def _zeqn(self,n,m,kind='h',forceRecompute=False):
        """Return parameters sufficient for specifying a Zernike term
        of desired order and azimuthal frequency.

        Given an order (or degree) n and azimuthal frequency f, and x-
        and y- rectangular (Cartesian) coordinates, produce parameters
        necessary for constructing the appropriate Zernike
        representation.

        An individual polynomial has the format:

        $$ Z_n^m = \sqrt{c} \Sigma^j\Sigma^k [a_{jk}X^jY^k] $$

        This function returns a tuple ($c$,cdict). $c$ is the square
        of the normalizing coefficient $\sqrt{c}$, and cdict contains
        key-value pairs (($j$,$k$),$a$), mapping the $X$ and $Y$
        exponents ($j$ and $k$, respectively) onto polynomial term
        coefficients ($a$). The resulting structure can be used to
        compute the wavefront height or slope for arbitrary pupil
        coordinates, or to generate string representations of the
        polynomials.

        Zernike terms are only defined when n and m have the same
        parity (both odd or both even).

        Please see Schwiegerling lecture notes in
        /doc/supporting_docs/ for eqn. references.

        Args:

          n (int): The Zernike order or degree.

          m (int): The azimuthal frequency.

          kind (str): 'h', 'dx', or 'dy', for height, partial x
              derivative (slope) or partial y derivative,
              respectively.

        Returns:

          params (tuple): (c,cdict), with c being the normalizing
              coefficient c and cdict being the map of exponent pairs
              onto inner coefficients.

        """
        absm = np.abs(m)

        kindIndex = self._kindDictionary[kind.lower()]

        if USE_CACHE_FILE:
            # open cache file in append mode:
            self._cacheHandle = file(self._cachefn,'a')

        # check that n and m are both even or both odd
        if (float(n-absm))%2.0:
            errString = 'zernike._zgen error: ' + \
                'parity of n and m are different; n = %d, m = %d'%(n,m)
            sys.exit(errString)

        # check that n is non-negative:
        if n<0:
            errString = 'zernike._zgen error: ' + \
                'n must be non-negative; n = %d'%n
            sys.exit(errString)

        # $|m|$ must be less than or equal to $n$.
        if abs(m)>n:
            errString = 'zernike._zgen error: ' + \
                '|m| must be less than or equal to n, but n=%d and m=%d.'%(n,m)
            sys.exit(errString)

        # These are the squares of the outer coefficients. It's useful
        # to keep them this way for _convertToString, since we'd
        # prefer to print the $\sqrt{}$ rather than a truncated irrational
        # number.
        if m==0:
            outerCoef = n+1
        else:
            outerCoef = 2*(n+1)

        srange = range((n-absm)/2+1)

        cdict = {}

        for s in srange:
            jrange = range(((n-absm)/2)-s+1)
            for j in jrange:

                # Subtract 1 from absm to determine range,
                # only when m<0.
                if m<0:
                    krange = range((absm-1)/2+1)
                else:
                    krange = range(absm/2+1)

                for k in krange:
                    # If m==0, k must also be 0;
                    # see eqn. 13c, 19c, and 20c, each of which
                    # only sum over s and j, not k.
                    if m==0:
                        assert(k==0)
                    # For m==0 cases, n/2 is used in coef denominator. Make
                    # sure that n is even, or else n/2 is not well-defined
                    # because n is an integer.
                    if m==0:
                        assert n%2==0

                    # Check to see if calculations are cached.
                    # If so, use cached values; if not, recalculate.
                    cached = self._termDict.has_key((n,m,kindIndex,s,j,k))

                    if cached and not forceRecompute:
                        t1,t2,t3,c,tXexp,tYexp = self._termDict[(n,m,kindIndex,s,j,k)]

                    else:

                        # The coefficient for each term in this
                        # polynomial has the format: $$\frac{t1n}{t1d1
                        # t1d2 t1d3} t2 t3$$. These six terms are
                        # computed here.
                        t1n = ((-1)**(s+k))*fact(n-s)
                        t1d1 = fact(s)
                        t1d2 = fact((n + absm)/2-s)
                        t1d3 = fact((n - absm)/2-s)
                        t1 = t1n/(t1d1*t1d2*t1d3)

                        t2 = choose((n - absm)/2 - s, j)
                        t3 = choose(absm, 2*k + (m<0))

                        if kind.lower()=='h':
                            # The (implied) coefficient of the $X^a Y^b$
                            # term at the end of eqns. 13a-c.
                            c = 1
                            tXexp = n - 2*(s+j+k) - (m<0)
                            tYexp = 2*(j+k) + (m<0)
                        elif kind.lower()=='dx':
                            # The coefficient of the $X^a Y^b$ term at
                            # the end of eqns. 19a-c.
                            c = (n - 2*(s+j+k) - (m<0))

                            # Could cacluate explicitly:
                            # $tXexp = X^{(n - 2*(s+j+k)- 1 - (m<0))}$
                            #
                            # However, piggy-backing on previous
                            # calculation of c speeds things up.
                            tXexp = c - 1
                            tYexp = 2*(j+k) + (m<0)

                        elif kind.lower()=='dy':
                            # The coefficient of the $X^a Y^b$ term at
                            # the end of eqns. 20a-c.
                            c = 2*(j+k) + (m<0)
                            tXexp = n - 2*(s+j+k) - (m<0)
                            tYexp = c - 1

                        else:
                            errString = 'zernike._zgen error: ' + \
                                'invalid kind \'%s\'; should be \'h\', \'dx\', or \'dy\'.'%kind
                            sys.exit(errString)

                        if not cached and USE_CACHE_FILE:
                            self._cacheHandle.write('%d\t'*12%(n,m,kindIndex,s,j,k,t1,t2,t3,c,tXexp,tYexp)+'\n')

                    ct123 = c*t1*t2*t3
                    # The key for the polynomial dictionary is the pair of X,Y
                    # coefficients.
                    termKey = (tXexp,tYexp)

                    # Leave this term out of the dictionary if its coefficient
                    # is 0.
                    if ct123:
                        # If we already have this term, add to its coefficient.
                        if cdict.has_key(termKey):
                            cdict[termKey] = cdict[termKey] + ct123
                        # If not, add it to the dictionary.
                        else:
                            cdict[termKey] = ct123

        # Remove zeros to speed up computations later.
        cdict = {key: value for key, value in cdict.items() if value}

        return (outerCoef,cdict)

    def _convertToString(self,params):
        """Return a string representation of a Zernike polynomial.

        This function takes a tuple, consisting of a squared
        normalizing coefficient and dictionary of inner coefficients
        and exponents, provided by _zeqn, and returns a string
        representation of the polynomial, with LaTeX- style markup.

        Example: a params of (10, {(3,4): 7, (2,5): -1}) would produce a
        two-term polynomial '\sqrt{10} [7 X^3 Y^4 - X^2 Y^5]', which could be used in LaTeX,
        pandoc, markdown, MathJax, or Word with MathType, to produce:

        $$ \sqrt{10} [7 X^3 Y^4 - X^2 Y^5] $$

        Args:

          params (tuple): A pair consisting of an outer coefficient
            $c$ and a dictionary mapping tuples (xexp,yexp) of
            exponents onto the corresponding term coefficients.

        Returns:

          string: A string representation of the polynomial.
        """
        c = params[0]
        cdict = params[1]

        keys = sorted(cdict.keys(), key=lambda tup: (tup[0]+tup[1],tup[0]))[::-1]

        outstr = ''
        firstKey = True
        for key in keys:
            coef = cdict[key]
            if coef>0:
                sign = '+'
            else:
                sign = '-'

            coef = abs(coef)

            if coef<0 or not firstKey:
                outstr = outstr + '%s'%sign
            if coef>1 or (key[0]==0 and key[1]==0):
                outstr = outstr + '%d'%coef
            if key[0]:
                outstr = outstr + 'X^{%d}'%key[0]
            if key[1]:
                outstr = outstr + 'Y^{%d}'%key[1]
            firstKey = False
            outstr = outstr + ' '

        outstr = outstr.strip()

        if np.sqrt(float(c))%1.0<.00001:
            cstr = '%d'%(np.sqrt(c))
        else:
            cstr = '\sqrt{%d}'%(c)

        if len(outstr):
            outstr = '%s [%s]'%(cstr,outstr)
        else:
            outstr = '%s'%(cstr)
        return outstr

    def _convertToSurface(self,params,X,Y,mask=None):
        """Return a phase map specified by a Zernike polynomial.

        This function takes a tuple, consisting of a squared
        normalizing coefficient and dictionary of inner coefficients
        and exponents, provided by _zeqn, and x- and y- rectangular
        (Cartesian) coordinates, and produces a phase map.

        This function works by evaluating the polynomial expressed by
        params at each coordinate specified by X and Y.

        Args:

          params (tuple): A pair consisting of an outer coefficient
            $c$ and a dictionary mapping tuples (xexp,yexp) of
            exponents onto the corresponding term coefficients.

          X (float): A scalar, vector, or matrix of X coordinates in unit pupil.

          Y (float): A scalar, vector, or matrix of Y coordinates in unit pupil.

          kind (str): 'h', 'dx', or 'dy', for height, partial x derivative (slope)
              or partial y derivative, respectively.

        Returns:

          float: height, dx, or dy; returned structure same size as X and Y.
        """

        # Check that shapes of X and Y are equal (not necessarily square).
        if not (X.shape[0]==Y.shape[0] and \
                    X.shape[1]==Y.shape[1]):
            errString = 'zernike.getSurface error: ' + \
                'X and Y must have the same shape, but X is %d x %d'%(X.shape[0],X.shape[1]) + \
                'and Y is %d x %d'%(Y.shape[0],Y.shape[1])
            sys.exit(errString)

        if mask is None:
            mask = np.ones(X.shape)

        params = self._zeqn(n,m,kind)
        normalizer = np.sqrt(params[0])

        matrix_out = np.zeros(X.shape)

        for item in params[1].items():
            matrix_out = matrix_out + item[1] * X**(item[0][0]) * Y**(item[0][1])

        matrix_out = matrix_out * np.sqrt(normalizer)
        matrix_out = matrix_out * mask

        return matrix_out

    def getSurface(self,n,m,X,Y,kind='h',mask=None):
        """Return a phase map specified by a Zernike order and azimuthal frequency.

        Given an order (or degree) n and azimuthal frequency f, and x- and y-
        rectangular (Cartesian) coordinates, produce a phase map of either height,
        partial x derivative, or partial y derivative.

        Zernike terms are only defined when n and m have the same parity (both odd
        or both even).

        The input X and Y values should be located inside a unit pupil, such that
        $$\sqrt{X^2 + Y^2}\leq 1$$

        Please see Schwiegerling lecture notes in /doc/supporting_docs/ for eqn.
        references.

        This function works by calling Zernike._zeqn to calculate the coefficients
        and exponents of the polynomial, and then using the supplied X and Y
        coordinates to produce the height map (or partial derivative).

        Args:

          n (int): The Zernike order or degree.

          m (int): The azimuthal frequency.

          X (float): A scalar, vector, or matrix of X coordinates in unit pupil.

          Y (float): A scalar, vector, or matrix of Y coordinates in unit pupil.

          kind (str): 'h', 'dx', or 'dy', for height, partial x derivative (slope)
              or partial y derivative, respectively.

        Returns:

          float: height, dx, or dy; returned structure same size as X and Y.
        """

        # Check that shapes of X and Y are equal (not necessarily square).
        if not np.all(X.shape==Y.shape):
            errString = 'zernike.getSurface error: ' + \
                'X and Y must have the same shape, but X is %d x %d'%(X.shape[0],X.shape[1]) + \
                'and Y is %d x %d'%(Y.shape[0],Y.shape[1])
            sys.exit(errString)

        if mask is None:
            mask = np.ones(X.shape)

        params = self._zeqn(n,m,kind)
        normalizer = np.sqrt(params[0])
        matrix_out = np.zeros(X.shape)

        for item in params[1].items():
            matrix_out = matrix_out + item[1] * X**(item[0][0]) * Y**(item[0][1])

        matrix_out = matrix_out * normalizer
        matrix_out = matrix_out * mask

        return matrix_out

    def getEquationString(self,n,m,kind='h',doubleDollar=False):
        """Return LaTeX-encoded of the Zernike polynomial specified by
        order n, frequency m.

        Args:

          n (int): The Zernike order or degree.

          m (int): The azimuthal frequency.

          kind (str): 'h', 'dx', or 'dy', for height, partial x
              derivative (slope) or partial y derivative,
              respectively.

          doubleDollar (bool): determines how to bookend the
              polynomial string; True causes bookending with '$$', to
              produce "display" math mode, whereas False would produce
              a string suitable for inline use.

        Returns:

          str: a LaTeX representation of the Zernike polynomial
              specified by n, m, and Kind.
        """

        params = self._zeqn(n,m,kind)
        rightString = self._convertToString(params)

        if kind.lower()=='h':
            leftString = 'Z^{%d}_{%d}'%(m,n)
        elif kind.lower()=='dx':
            leftString = '\\frac{\delta Z^{%d}_{%d}}{\delta x}'%(m,n)
        elif kind.lower()=='dy':
            leftString = '\\frac{\delta Z^{%d}_{%d}}{\delta y}'%(m,n)
        else:
            sys.exit('zernike.getEquationString: invalid kind %s'%kind)

        if doubleDollar:
            bookend = '$$'
        else:
            bookend = '$'

        return '%s %s = %s %s'%(bookend,leftString,rightString,bookend)

    def plotPolynomial(self,n,m,kind='h'):
        """Plot a polynomial surface specified by order n, frequency m, and kind.

        Args:

          n (int): The Zernike order or degree.

          m (int): The azimuthal frequency.

          kind (str): 'h', 'dx', or 'dy', for height, partial x
              derivative (slope) or partial y derivative,
              respectively.

        Calling function/script required to provide a plotting context (e.g. pyplot.figure).
        """

        from mpl_toolkits.mplot3d import Axes3D

        N = 64
        mask = np.zeros((N,N))
        xx,yy = np.meshgrid(np.linspace(-1,1,N),np.linspace(-1,1,N))
        d = np.sqrt(xx**2 + yy**2)
        mask[np.where(d<1)] = 1
        surface = self.getSurface(n,m,xx,yy,kind,mask)

        surface = surface * mask
        #plt.figure()
        ax = plt.axes([0,.2,1,.8],projection='3d')
        surf = ax.plot_wireframe(xx,yy,surface,rstride=1,cstride=1,color='k')
        ax.view_init(elev=70., azim=40)

        eqstr = self.getEquationString(n,m,kind)
        eqstr = splitEquation(eqstr,160,'$')
        print 'plotting %s'%eqstr
        plt.axes([0,0,1,.2])
        plt.xticks([])
        plt.yticks([])
        plt.box('off')
        fontsize = 12
        plt.text(0.5,0.5,eqstr,ha='center',va='center',fontsize=fontsize)


from pgmpy.estimators import BaseEstimator
from pgmpy.factors import TabularCPD
from pgmpy.models import BayesianModel
import numpy as np

class MaximumLikelihoodEstimator(BaseEstimator):
    """
    Class used to compute parameters for a model using Maximum Likelihood Estimate.

    Parameters
    ----------
    model: pgmpy.models.BayesianModel or pgmpy.models.MarkovModel or pgmpy.models.NoisyOrModel
        model for which parameter estimation is to be done

    data: pandas DataFrame object
        datafame object with column names same as the variable names of the network

    Examples
    --------
    >>> import numpy as np
    >>> import pandas as pd
    >>> from pgmpy.models import BayesianModel
    >>> from pgmpy.estimators import MaximumLikelihoodEstimator
    >>> values = pd.DataFrame(np.random.randint(low=0, high=2, size=(1000, 5)),
    ...                       columns=['A', 'B', 'C', 'D', 'E'])
    >>> model = BayesianModel([('A', 'B'), ('C', 'B'), ('C', 'D'), ('B', 'E')])
    >>> estimator = MaximumLikelihoodEstimator(model, values)
    """
    def __init__(self, model, data):
        if not isinstance(model, BayesianModel):
            raise NotImplementedError("Maximum Likelihood Estimate is only implemented of BayesianModel")

        super().__init__(model, data)

    def get_parameters(self):
        """
        Method used to get parameters.

        Returns
        -------
        parameters: list
            List containing all the parameters. For Bayesian Model it would be list of CPDs'
            for Markov Model it would be a list of factors

        Examples
        --------
        >>> import numpy as np
        >>> import pandas as pd
        >>> from pgmpy.models import BayesianModel
        >>> from pgmpy.estimators import MaximumLikelihoodEstimator
        >>> values = pd.DataFrame(np.random.randint(low=0, high=2, size=(1000, 5)),
        ...                       columns=['A', 'B', 'C', 'D', 'E'])
        >>> model = BayesianModel([('A', 'B'), ('C', 'B'), ('C', 'D'), ('B', 'E')])
        >>> estimator = MaximumLikelihoodEstimator(model, values)
        >>> estimator.get_parameters()
        """
        parameters = []

        for node in self.model.nodes():
            parents = self.model.get_parents(node)
            if not parents:
                state_counts = self.data.ix[:, node].value_counts()
                cpd = TabularCPD(node, self.node_card[node],
                                 state_counts.values[:, np.newaxis])
                cpd.normalize()
                parameters.append(cpd)
            else:
                parent_card = np.array([self.node_card[parent] for parent in parents])
                var_card = self.node_card[node]
                state_counts = self.data.groupby([node] + self.model.predecessors(node)).count()
                values = state_counts.iloc[:, 0].reshape(var_card,
                                                         np.product(parent_card))
                cpd = TabularCPD(node, var_card, values,
                                 evidence=parents,
                                 evidence_card=parent_card.astype('int'))
                cpd.normalize()
                parameters.append(cpd)

        return parameters


"""
============================================================
Single estimator versus bagging: bias-variance decomposition
============================================================

This example illustrates and compares the bias-variance decomposition of the
expected mean squared error of a single estimator against a bagging ensemble.

In regression, the expected mean squared error of an estimator can be
decomposed in terms of bias, variance and noise. On average over datasets of
the regression problem, the bias term measures the average amount by which the
predictions of the estimator differ from the predictions of the best possible
estimator for the problem (i.e., the Bayes model). The variance term measures
the variability of the predictions of the estimator when fit over different
instances LS of the problem. Finally, the noise measures the irreducible part
of the error which is due the variability in the data.

The upper left figure illustrates the predictions (in dark red) of a single
decision tree trained over a random dataset LS (the blue dots) of a toy 1d
regression problem. It also illustrates the predictions (in light red) of other
single decision trees trained over other (and different) randomly drawn
instances LS of the problem. Intuitively, the variance term here corresponds to
the width of the beam of predictions (in light red) of the individual
estimators. The larger the variance, the more sensitive are the predictions for
`x` to small changes in the training set. The bias term corresponds to the
difference between the average prediction of the estimator (in cyan) and the
best possible model (in dark blue). On this problem, we can thus observe that
the bias is quite low (both the cyan and the blue curves are close to each
other) while the variance is large (the red beam is rather wide).

The lower left figure plots the pointwise decomposition of the expected mean
squared error of a single decision tree. It confirms that the bias term (in
blue) is low while the variance is large (in green). It also illustrates the
noise part of the error which, as expected, appears to be constant and around
`0.01`.

The right figures correspond to the same plots but using instead a bagging
ensemble of decision trees. In both figures, we can observe that the bias term
is larger than in the previous case. In the upper right figure, the difference
between the average prediction (in cyan) and the best possible model is larger
(e.g., notice the offset around `x=2`). In the lower right figure, the bias
curve is also slightly higher than in the lower left figure. In terms of
variance however, the beam of predictions is narrower, which suggests that the
variance is lower. Indeed, as the lower right figure confirms, the variance
term (in green) is lower than for single decision trees. Overall, the bias-
variance decomposition is therefore no longer the same. The tradeoff is better
for bagging: averaging several decision trees fit on bootstrap copies of the
dataset slightly increases the bias term but allows for a larger reduction of
the variance, which results in a lower overall mean squared error (compare the
red curves int the lower figures). The script output also confirms this
intuition. The total error of the bagging ensemble is lower than the total
error of a single decision tree, and this difference indeed mainly stems from a
reduced variance.

For further details on bias-variance decomposition, see section 7.3 of [1]_.

References
----------

.. [1] T. Hastie, R. Tibshirani and J. Friedman,
       "Elements of Statistical Learning", Springer, 2009.

"""
print(__doc__)

# Author: Gilles Louppe <g.louppe@gmail.com>
# License: BSD 3 clause

import numpy as np
from matplotlib import pyplot as plt

from sklearn.ensemble import BaggingRegressor
from sklearn.tree import DecisionTreeRegressor

# Settings
n_repeat = 50       # Number of iterations for computing expectations
n_train = 50        # Size of the training set
n_test = 1000       # Size of the test set
noise = 0.1         # Standard deviation of the noise
np.random.seed(0)

# Change this for exploring the bias-variance decomposition of other
# estimators. This should work well for estimators with high variance (e.g.,
# decision trees or KNN), but poorly for estimators with low variance (e.g.,
# linear models).
estimators = [("Tree", DecisionTreeRegressor()),
              ("Bagging(Tree)", BaggingRegressor(DecisionTreeRegressor()))]

n_estimators = len(estimators)

# Generate data
def f(x):
    x = x.ravel()

    return np.exp(-x ** 2) + 1.5 * np.exp(-(x - 2) ** 2)

def generate(n_samples, noise, n_repeat=1):
    X = np.random.rand(n_samples) * 10 - 5
    X = np.sort(X)

    if n_repeat == 1:
        y = f(X) + np.random.normal(0.0, noise, n_samples)
    else:
        y = np.zeros((n_samples, n_repeat))

        for i in range(n_repeat):
            y[:, i] = f(X) + np.random.normal(0.0, noise, n_samples)

    X = X.reshape((n_samples, 1))

    return X, y

X_train = []
y_train = []

for i in range(n_repeat):
    X, y = generate(n_samples=n_train, noise=noise)
    X_train.append(X)
    y_train.append(y)

X_test, y_test = generate(n_samples=n_test, noise=noise, n_repeat=n_repeat)

# Loop over estimators to compare
for n, (name, estimator) in enumerate(estimators):
    # Compute predictions
    y_predict = np.zeros((n_test, n_repeat))

    for i in xrange(n_repeat):
        estimator.fit(X_train[i], y_train[i])
        y_predict[:, i] = estimator.predict(X_test)

    # Bias^2 + Variance + Noise decomposition of the mean squared error
    y_error = np.zeros(n_test)

    for i in range(n_repeat):
        for j in range(n_repeat):
            y_error += (y_test[:, j] - y_predict[:, i]) ** 2

    y_error /= (n_repeat * n_repeat)

    y_noise = np.var(y_test, axis=1)
    y_bias = (f(X_test) - np.mean(y_predict, axis=1)) ** 2
    y_var = np.var(y_predict, axis=1)

    print("{0}: {1:.4f} (error) = {2:.4f} (bias^2) "
          " + {3:.4f} (var) + {4:.4f} (noise)".format(name,
                                                      np.mean(y_error),
                                                      np.mean(y_bias),
                                                      np.mean(y_var),
                                                      np.mean(y_noise)))

    # Plot figures
    plt.subplot(2, n_estimators, n + 1)
    plt.plot(X_test, f(X_test), "b", label="$f(x)$")
    plt.plot(X_train[0], y_train[0], ".b", label="LS ~ $y = f(x)+noise$")

    for i in range(n_repeat):
        if i == 0:
            plt.plot(X_test, y_predict[:, i], "r", label="$\^y(x)$")
        else:
            plt.plot(X_test, y_predict[:, i], "r", alpha=0.05)

    plt.plot(X_test, np.mean(y_predict, axis=1), "c",
             label="$\mathbb{E}_{LS} \^y(x)$")

    plt.xlim([-5, 5])
    plt.title(name)

    if n == 0:
        plt.legend(loc="upper left", prop={"size": 11})

    plt.subplot(2, n_estimators, n_estimators + n + 1)
    plt.plot(X_test, y_error, "r", label="$error(x)$")
    plt.plot(X_test, y_bias, "b", label="$bias^2(x)$"),
    plt.plot(X_test, y_var, "g", label="$variance(x)$"),
    plt.plot(X_test, y_noise, "c", label="$noise(x)$")

    plt.xlim([-5, 5])
    plt.ylim([0, 0.1])

    if n == 0:
        plt.legend(loc="upper left", prop={"size": 11})

plt.show()


from datetime import datetime, timedelta
from pytz import timezone
import warnings

import pandas as pd

import pytest
from numpy.testing import assert_allclose

from conftest import requires_siphon, has_siphon, skip_windows

pytestmark = pytest.mark.skipif(not has_siphon, reason='requires siphon')

if has_siphon:
    with warnings.catch_warnings():
        # don't emit import warning
        warnings.simplefilter("ignore")
        from pvlib.forecast import GFS, HRRR_ESRL, HRRR, NAM, NDFD, RAP

    # setup times and location to be tested. Tucson, AZ
    _latitude = 32.2
    _longitude = -110.9
    _tz = 'US/Arizona'
    _start = pd.Timestamp.now(tz=_tz)
    _end = _start + pd.Timedelta(days=1)
    _modelclasses = [
        GFS, NAM, HRRR, NDFD, RAP,
        pytest.param(
            HRRR_ESRL, marks=[
                skip_windows,
                pytest.mark.xfail(reason="HRRR_ESRL is unreliable"),
                pytest.mark.timeout(timeout=60),
                pytest.mark.filterwarnings('ignore:.*experimental')])]
    _working_models = []
    _variables = ['temp_air', 'wind_speed', 'total_clouds', 'low_clouds',
                  'mid_clouds', 'high_clouds', 'dni', 'dhi', 'ghi']
    _nonnan_variables = ['temp_air', 'wind_speed', 'total_clouds', 'dni',
                         'dhi', 'ghi']
else:
    _modelclasses = []

# make a model object for each model class
# get the data for that model and store it in an
# attribute for further testing
@requires_siphon
@pytest.fixture(scope='module', params=_modelclasses)
def model(request):
    amodel = request.param()
    try:
        raw_data = amodel.get_data(_latitude, _longitude, _start, _end)
    except Exception as e:
        warnings.warn('Exception getting data for {}.\n'
                      'latitude, longitude, start, end = {} {} {} {}\n{}'
                      .format(amodel, _latitude, _longitude, _start, _end, e))
        raw_data = pd.DataFrame()  # raw_data.empty will be used later
    amodel.raw_data = raw_data
    return amodel

@requires_siphon
def test_process_data(model):
    for how in ['liujordan', 'clearsky_scaling']:
        if model.raw_data.empty:
            warnings.warn('Could not test {} process_data with how={} '
                          'because raw_data was empty'.format(model, how))
            continue
        data = model.process_data(model.raw_data, how=how)
        for variable in _nonnan_variables:
            try:
                assert not data[variable].isnull().values.any()
            except AssertionError:
                warnings.warn('{}, {}, data contained null values'
                              .format(model, variable))

@requires_siphon
def test_bad_kwarg_get_data():
    # For more information on why you would want to pass an unknown keyword
    # argument, see Github issue #745.
    amodel = NAM()
    data = amodel.get_data(_latitude, _longitude, _start, _end,
                           bad_kwarg=False)
    assert not data.empty

@requires_siphon
def test_bad_kwarg_get_processed_data():
    # For more information on why you would want to pass an unknown keyword
    # argument, see Github issue #745.
    amodel = NAM()
    data = amodel.get_processed_data(_latitude, _longitude, _start, _end,
                                     bad_kwarg=False)
    assert not data.empty

@requires_siphon
def test_how_kwarg_get_processed_data():
    amodel = NAM()
    data = amodel.get_processed_data(_latitude, _longitude, _start, _end,
                                     how='clearsky_scaling')
    assert not data.empty

@requires_siphon
def test_vert_level():
    amodel = NAM()
    vert_level = 5000
    amodel.get_processed_data(_latitude, _longitude, _start, _end,
                              vert_level=vert_level)

@requires_siphon
def test_datetime():
    amodel = NAM()
    start = datetime.now()
    end = start + timedelta(days=1)
    amodel.get_processed_data(_latitude, _longitude, start, end)

@requires_siphon
def test_queryvariables():
    amodel = GFS()
    new_variables = ['u-component_of_wind_height_above_ground']
    data = amodel.get_data(_latitude, _longitude, _start, _end,
                           query_variables=new_variables)
    data['u-component_of_wind_height_above_ground']

@requires_siphon
def test_latest():
    GFS(set_type='latest')

@requires_siphon
def test_full():
    GFS(set_type='full')

@requires_siphon
def test_temp_convert():
    amodel = GFS()
    data = pd.DataFrame({'temp_air': [273.15]})
    data['temp_air'] = amodel.kelvin_to_celsius(data['temp_air'])

    assert_allclose(data['temp_air'].values, 0.0)

# @requires_siphon
# def test_bounding_box():
#     amodel = GFS()
#     latitude = [31.2,32.2]
#     longitude = [-111.9,-110.9]
#     new_variables = {'temperature':'Temperature_surface'}
#     data = amodel.get_query_data(latitude, longitude, _start, _end,
#                                  variables=new_variables)

@requires_siphon
def test_set_location():
    amodel = GFS()
    latitude, longitude = 32.2, -110.9
    time = datetime.now(timezone('UTC'))
    amodel.set_location(time, latitude, longitude)

def test_cloud_cover_to_transmittance_linear():
    amodel = GFS()
    assert_allclose(amodel.cloud_cover_to_transmittance_linear(0), 0.75)
    assert_allclose(amodel.cloud_cover_to_transmittance_linear(100), 0.0)
    assert_allclose(amodel.cloud_cover_to_transmittance_linear(0, 0.5), 0.5)

def test_cloud_cover_to_ghi_linear():
    amodel = GFS()
    ghi_clear = 1000
    offset = 25
    out = amodel.cloud_cover_to_ghi_linear(0, ghi_clear, offset=offset)
    assert_allclose(out, 1000)
    out = amodel.cloud_cover_to_ghi_linear(100, ghi_clear, offset=offset)
    assert_allclose(out, 250)


import numpy as np
import pandas as pd
from bokeh.plotting import figure, show, output_file

# Generate some synthetic time series for six different categories
cats = list("abcdef")
yy = np.random.randn(2000)
g = np.random.choice(cats, 2000)
for i, l in enumerate(cats):
    yy[g == l] += i // 2
df = pd.DataFrame(dict(score=yy, group=g))

# Find the quartiles and IQR foor each category
groups = df.groupby('group')
q1 = groups.quantile(q=0.25)
q2 = groups.quantile(q=0.5)
q3 = groups.quantile(q=0.75)
iqr = q3 - q1
upper = q3 + 1.5*iqr
lower = q1 - 1.5*iqr

# find the outliers for each category
def outliers(group):
    cat = group.name
    return group[(group.score > upper.loc[cat][0]) | (group.score < lower.loc[cat][0])]['score']
out = groups.apply(outliers).dropna()

# Prepare outlier data for plotting, we need coordinate for every outlier.
outx = []
outy = []
for cat in cats:
    # only add outliers if they exist
    if not out.loc[cat].empty:
        for value in out[cat]:
            outx.append(cat)
            outy.append(value)

output_file("boxplot.html")

p = figure(tools="save", background_fill="#EFE8E2", title="", x_range=cats)

# If no outliers, shrink lengths of stems to be no longer than the minimums or maximums
qmin = groups.quantile(q=0.00)
qmax = groups.quantile(q=1.00)
upper.score = [min([x,y]) for (x,y) in zip(list(qmax.iloc[:,0]),upper.score) ]
lower.score = [max([x,y]) for (x,y) in zip(list(qmin.iloc[:,0]),lower.score) ]

# stems
p.segment(cats, upper.score, cats, q3.score, line_width=2, line_color="black")
p.segment(cats, lower.score, cats, q1.score, line_width=2, line_color="black")

# boxes
p.rect(cats, (q3.score+q2.score)/2, 0.7, q3.score-q2.score,
    fill_color="#E08E79", line_width=2, line_color="black")
p.rect(cats, (q2.score+q1.score)/2, 0.7, q2.score-q1.score,
    fill_color="#3B8686", line_width=2, line_color="black")

# whiskers (almost-0 height rects simpler than segments)
p.rect(cats, lower.score, 0.2, 0.01, line_color="black")
p.rect(cats, upper.score, 0.2, 0.01, line_color="black")

# outliers
p.circle(outx, outy, size=6, color="#F38630", fill_alpha=0.6)

p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = "white"
p.grid.grid_line_width = 2
p.xaxis.major_label_text_font_size="12pt"

show(p)


from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt
from GProtation import make_plot, lnprob, neglnlike
import emcee
import time
import george
from george.kernels import ExpSquaredKernel, ExpSine2Kernel
import scipy.optimize as spo

def GP_periodogram(x, y, yerr, p_init, plims, N):
	"""
	This function takes a light curves and attempts to produce a GP periodogram.
	It returns the value of the highest peak.
	The kernel hyperparameters are optimised over a grid of periods.
	This is also a "profile likelihood".
	x, y, yerr: the light curve.
	p_init: the initial guess for the period.
	plims: the (log) boundaries for the grid.
	N: the number of grid points.
	"""

	# create the grid
	periods = np.linspace(np.exp(plims[0], np.exp(plims[1], 10)

	# initial hyperparameters

if __name__ == "__main__":

	# fake data
	x = np.arange(0, 10, 100)
	p = 2
	err = .1
	y = np.sin(2*np.pi*(1./p)*x) + np.random.randn(100)*err
	yerr = np.ones_like(y) * err
	p_init, plims = 2, np.log(.1, 5)

	GP_periodogram(x, y, yerr, p_init, plims, 10)


import numpy
#import scipy
import matplotlib.pyplot
import math

def weibullGenerator(scale, shape, start, end, step):
    '''
    Generates Weibull sample lists per the parameters.
    Returns two lists of X and Y values distributed per Weibull.
    '''

    weibullSamplesY = []
    weibullSamplesX = []
    for i in numpy.arange(start, end, step):
        weibullSamplesY.append((shape/scale)*((i/scale)**(shape-1.0))*(math.exp(-(i/scale)**shape)))
        weibullSamplesX.append(i)
    return weibullSamplesX, weibullSamplesY

def readValuesFromFile(filename):
    '''
    Reads values from a file and returns a list of floats.
    '''
    yValues = [] # Y values to be read from file.
    fileHandle = open(filename, 'r') # Opens file for reading.
    #yValues= list(fileHandle) # Read all values into yValues.
    for line in fileHandle:
        yValues.append(float(line.rstrip()))
    fileHandle.close()
    return yValues

# Now construct a map of parameters per Weibull samples.
# Key is filename with samples, value is list of lists:
# list1 is weibullGenerator parameters to generate a Weibull graph from matplotlib,
# list2 is set of parameters to plot the samples from the filename.
weibullParameterMap = {
    'weibull_Scale1.0_Shape5.0.csv': [[1.0,5.0,0.0,3.0,.01], [0.0,3.0,0.0,2.0]],
    'weibull_Scale1.0_Shape1.0.csv': [[1.0,1.0,0.0,3.0,.01], [0.0,3.0,0.0,1.0]],
    'weibull_Scale1.0_Shape2.0.csv': [[1.0,2.0,0.0,3.0,.01], [0.0,3.0,0.0,0.9]],
    'weibull_Scale1.0_Shape0.5.csv': [[1.0,0.5,0.0,3.0,.01], [0.0,3.0,0.0,5.0]]
}

# Iterate through dictionary and generate graphs.
for filename, parameters in weibullParameterMap.items():
    weibullSamplesX, weibullSamplesY = weibullGenerator(parameters[0][0], parameters[0][1],
                                                        parameters[0][2], parameters[0][3],
                                                        parameters[0][4])
    print("Parameters: ", parameters)
    matplotlib.pyplot.figure()
    matplotlib.pyplot.plot(weibullSamplesX, weibullSamplesY)
    matplotlib.pyplot.grid(True)
    matplotlib.pyplot.xlabel("x values")
    matplotlib.pyplot.ylabel("Probability")
    matplotlib.pyplot.title('Weibull pdf: ' + str(parameters[0]))
    matplotlib.pyplot.savefig(filename + '_pdf.png')
    matplotlib.pyplot.show()
    #matplotlib.pyplot.close()
    ySamples = readValuesFromFile(filename)
    matplotlib.pyplot.figure()
    matplotlib.pyplot.hist(ySamples, bins=300, range=(parameters[1][0], parameters[1][1]),
                                                       normed=True, color='r')
    matplotlib.pyplot.axis(parameters[1])
    matplotlib.pyplot.grid(True)
    matplotlib.pyplot.xlabel("x values")
    matplotlib.pyplot.ylabel("Probability")
    matplotlib.pyplot.title('Samples from ' + filename)
    matplotlib.pyplot.savefig(filename + '_sample.png')
    matplotlib.pyplot.show()
    #matplotlib.pyplot.close()
print("*** Done! ***")

##print (weibullGenerator(1.0,5.0,0.0,3.0,.001))
#weibullSamplesX, weibullSamplesY = weibullGenerator(1.0,5.0,0.0,3.0,.001)
#matplotlib.pyplot.figure()
##matplotlib.pyplot.hist(weibullSamplesY, 100)
#matplotlib.pyplot.plot(weibullSamplesX, weibullSamplesY)
#matplotlib.pyplot.grid(True)
#matplotlib.pyplot.show()
##matplotlib.pyplot.close()
#print("Done Figure 1.")
#ySamples = readValuesFromFile('weibull_Scale1.0_Shape5.0.csv')
#print("Done reading.")
##print (ySamples)
#matplotlib.pyplot.figure()
#matplotlib.pyplot.hist(ySamples, 500)
#matplotlib.pyplot.axis([0.0,3.0,0.0,650])
#matplotlib.pyplot.grid(True)
#matplotlib.pyplot.show()
##matplotlib.pyplot.close()
#print("Done Figure 2.")


# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Fabian Pedregosa <fabian.pedregosa@inria.fr>
#
# License: BSD 3 clause

import numpy as np
from scipy import sparse
from scipy import linalg
from itertools import product

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import ignore_warnings

from sklearn.linear_model.base import LinearRegression
from sklearn.linear_model.base import _preprocess_data
from sklearn.linear_model.base import sparse_center_data, center_data
from sklearn.linear_model.base import _rescale_data
from sklearn.utils import check_random_state
from sklearn.utils.testing import assert_greater
from sklearn.datasets.samples_generator import make_sparse_uncorrelated
from sklearn.datasets.samples_generator import make_regression

rng = np.random.RandomState(0)

def test_linear_regression():
    # Test LinearRegression on a simple dataset.
    # a simple dataset
    X = [[1], [2]]
    Y = [1, 2]

    reg = LinearRegression()
    reg.fit(X, Y)

    assert_array_almost_equal(reg.coef_, [1])
    assert_array_almost_equal(reg.intercept_, [0])
    assert_array_almost_equal(reg.predict(X), [1, 2])

    # test it also for degenerate input
    X = [[1]]
    Y = [0]

    reg = LinearRegression()
    reg.fit(X, Y)
    assert_array_almost_equal(reg.coef_, [0])
    assert_array_almost_equal(reg.intercept_, [0])
    assert_array_almost_equal(reg.predict(X), [0])

def test_linear_regression_sample_weights():
    # TODO: loop over sparse data as well

    rng = np.random.RandomState(0)

    # It would not work with under-determined systems
    for n_samples, n_features in ((6, 5), ):

        y = rng.randn(n_samples)
        X = rng.randn(n_samples, n_features)
        sample_weight = 1.0 + rng.rand(n_samples)

        for intercept in (True, False):

            # LinearRegression with explicit sample_weight
            reg = LinearRegression(fit_intercept=intercept)
            reg.fit(X, y, sample_weight=sample_weight)
            coefs1 = reg.coef_
            inter1 = reg.intercept_

            assert_equal(reg.coef_.shape, (X.shape[1], ))  # sanity checks
            assert_greater(reg.score(X, y), 0.5)

            # Closed form of the weighted least square
            # theta = (X^T W X)^(-1) * X^T W y
            W = np.diag(sample_weight)
            if intercept is False:
                X_aug = X
            else:
                dummy_column = np.ones(shape=(n_samples, 1))
                X_aug = np.concatenate((dummy_column, X), axis=1)

            coefs2 = linalg.solve(X_aug.T.dot(W).dot(X_aug),
                                  X_aug.T.dot(W).dot(y))

            if intercept is False:
                assert_array_almost_equal(coefs1, coefs2)
            else:
                assert_array_almost_equal(coefs1, coefs2[1:])
                assert_almost_equal(inter1, coefs2[0])

def test_raises_value_error_if_sample_weights_greater_than_1d():
    # Sample weights must be either scalar or 1D

    n_sampless = [2, 3]
    n_featuress = [3, 2]

    for n_samples, n_features in zip(n_sampless, n_featuress):
        X = rng.randn(n_samples, n_features)
        y = rng.randn(n_samples)
        sample_weights_OK = rng.randn(n_samples) ** 2 + 1
        sample_weights_OK_1 = 1.
        sample_weights_OK_2 = 2.

        reg = LinearRegression()

        # make sure the "OK" sample weights actually work
        reg.fit(X, y, sample_weights_OK)
        reg.fit(X, y, sample_weights_OK_1)
        reg.fit(X, y, sample_weights_OK_2)

def test_fit_intercept():
    # Test assertions on betas shape.
    X2 = np.array([[0.38349978, 0.61650022],
                   [0.58853682, 0.41146318]])
    X3 = np.array([[0.27677969, 0.70693172, 0.01628859],
                   [0.08385139, 0.20692515, 0.70922346]])
    y = np.array([1, 1])

    lr2_without_intercept = LinearRegression(fit_intercept=False).fit(X2, y)
    lr2_with_intercept = LinearRegression(fit_intercept=True).fit(X2, y)

    lr3_without_intercept = LinearRegression(fit_intercept=False).fit(X3, y)
    lr3_with_intercept = LinearRegression(fit_intercept=True).fit(X3, y)

    assert_equal(lr2_with_intercept.coef_.shape,
                 lr2_without_intercept.coef_.shape)
    assert_equal(lr3_with_intercept.coef_.shape,
                 lr3_without_intercept.coef_.shape)
    assert_equal(lr2_without_intercept.coef_.ndim,
                 lr3_without_intercept.coef_.ndim)

def test_linear_regression_sparse(random_state=0):
    # Test that linear regression also works with sparse data
    random_state = check_random_state(random_state)
    for i in range(10):
        n = 100
        X = sparse.eye(n, n)
        beta = random_state.rand(n)
        y = X * beta[:, np.newaxis]

        ols = LinearRegression()
        ols.fit(X, y.ravel())
        assert_array_almost_equal(beta, ols.coef_ + ols.intercept_)

        assert_array_almost_equal(ols.predict(X) - y.ravel(), 0)

def test_linear_regression_multiple_outcome(random_state=0):
    # Test multiple-outcome linear regressions
    X, y = make_regression(random_state=random_state)

    Y = np.vstack((y, y)).T
    n_features = X.shape[1]

    reg = LinearRegression(fit_intercept=True)
    reg.fit((X), Y)
    assert_equal(reg.coef_.shape, (2, n_features))
    Y_pred = reg.predict(X)
    reg.fit(X, y)
    y_pred = reg.predict(X)
    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3)

def test_linear_regression_sparse_multiple_outcome(random_state=0):
    # Test multiple-outcome linear regressions with sparse data
    random_state = check_random_state(random_state)
    X, y = make_sparse_uncorrelated(random_state=random_state)
    X = sparse.coo_matrix(X)
    Y = np.vstack((y, y)).T
    n_features = X.shape[1]

    ols = LinearRegression()
    ols.fit(X, Y)
    assert_equal(ols.coef_.shape, (2, n_features))
    Y_pred = ols.predict(X)
    ols.fit(X, y.ravel())
    y_pred = ols.predict(X)
    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3)

def test_preprocess_data():
    n_samples = 200
    n_features = 2
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)
    expected_X_mean = np.mean(X, axis=0)
    expected_X_norm = np.std(X, axis=0) * np.sqrt(X.shape[0])
    expected_y_mean = np.mean(y, axis=0)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=False, normalize=False)
    assert_array_almost_equal(X_mean, np.zeros(n_features))
    assert_array_almost_equal(y_mean, 0)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt, X)
    assert_array_almost_equal(yt, y)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=False)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt, X - expected_X_mean)
    assert_array_almost_equal(yt, y - expected_y_mean)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=True)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, expected_X_norm)
    assert_array_almost_equal(Xt, (X - expected_X_mean) / expected_X_norm)
    assert_array_almost_equal(yt, y - expected_y_mean)

def test_preprocess_data_multioutput():
    n_samples = 200
    n_features = 3
    n_outputs = 2
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples, n_outputs)
    expected_y_mean = np.mean(y, axis=0)

    args = [X, sparse.csc_matrix(X)]
    for X in args:
        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=False,
                                               normalize=False)
        assert_array_almost_equal(y_mean, np.zeros(n_outputs))
        assert_array_almost_equal(yt, y)

        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=True,
                                               normalize=False)
        assert_array_almost_equal(y_mean, expected_y_mean)
        assert_array_almost_equal(yt, y - y_mean)

        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=True,
                                               normalize=True)
        assert_array_almost_equal(y_mean, expected_y_mean)
        assert_array_almost_equal(yt, y - y_mean)

def test_preprocess_data_weighted():
    n_samples = 200
    n_features = 2
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)
    sample_weight = rng.rand(n_samples)
    expected_X_mean = np.average(X, axis=0, weights=sample_weight)
    expected_y_mean = np.average(y, axis=0, weights=sample_weight)

    # XXX: if normalize=True, should we expect a weighted standard deviation?
    #      Currently not weighted, but calculated with respect to weighted mean
    expected_X_norm = (np.sqrt(X.shape[0]) *
                       np.mean((X - expected_X_mean) ** 2, axis=0) ** .5)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=False,
                         sample_weight=sample_weight)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt, X - expected_X_mean)
    assert_array_almost_equal(yt, y - expected_y_mean)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=True,
                         sample_weight=sample_weight)
    assert_array_almost_equal(X_mean, expected_X_mean)
    assert_array_almost_equal(y_mean, expected_y_mean)
    assert_array_almost_equal(X_norm, expected_X_norm)
    assert_array_almost_equal(Xt, (X - expected_X_mean) / expected_X_norm)
    assert_array_almost_equal(yt, y - expected_y_mean)

def test_sparse_preprocess_data_with_return_mean():
    n_samples = 200
    n_features = 2
    # random_state not supported yet in sparse.rand
    X = sparse.rand(n_samples, n_features, density=.5)  # , random_state=rng
    X = X.tolil()
    y = rng.rand(n_samples)
    XA = X.toarray()
    expected_X_norm = np.std(XA, axis=0) * np.sqrt(X.shape[0])

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=False, normalize=False,
                         return_mean=True)
    assert_array_almost_equal(X_mean, np.zeros(n_features))
    assert_array_almost_equal(y_mean, 0)
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt.A, XA)
    assert_array_almost_equal(yt, y)

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=False,
                         return_mean=True)
    assert_array_almost_equal(X_mean, np.mean(XA, axis=0))
    assert_array_almost_equal(y_mean, np.mean(y, axis=0))
    assert_array_almost_equal(X_norm, np.ones(n_features))
    assert_array_almost_equal(Xt.A, XA)
    assert_array_almost_equal(yt, y - np.mean(y, axis=0))

    Xt, yt, X_mean, y_mean, X_norm = \
        _preprocess_data(X, y, fit_intercept=True, normalize=True,
                         return_mean=True)
    assert_array_almost_equal(X_mean, np.mean(XA, axis=0))
    assert_array_almost_equal(y_mean, np.mean(y, axis=0))
    assert_array_almost_equal(X_norm, expected_X_norm)
    assert_array_almost_equal(Xt.A, XA / expected_X_norm)
    assert_array_almost_equal(yt, y - np.mean(y, axis=0))

def test_csr_preprocess_data():
    # Test output format of _preprocess_data, when input is csr
    X, y = make_regression()
    X[X < 2.5] = 0.0
    csr = sparse.csr_matrix(X)
    csr_, y, _, _, _ = _preprocess_data(csr, y, True)
    assert_equal(csr_.getformat(), 'csr')

def test_rescale_data():
    n_samples = 200
    n_features = 2

    sample_weight = 1.0 + rng.rand(n_samples)
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)
    rescaled_X, rescaled_y = _rescale_data(X, y, sample_weight)
    rescaled_X2 = X * np.sqrt(sample_weight)[:, np.newaxis]
    rescaled_y2 = y * np.sqrt(sample_weight)
    assert_array_almost_equal(rescaled_X, rescaled_X2)
    assert_array_almost_equal(rescaled_y, rescaled_y2)

@ignore_warnings  # all deprecation warnings
def test_deprecation_center_data():
    n_samples = 200
    n_features = 2

    w = 1.0 + rng.rand(n_samples)
    X = rng.rand(n_samples, n_features)
    y = rng.rand(n_samples)

    param_grid = product([True, False], [True, False], [True, False],
                         [None, w])

    for (fit_intercept, normalize, copy, sample_weight) in param_grid:

        XX = X.copy()  # such that we can try copy=False as well

        X1, y1, X1_mean, X1_var, y1_mean = \
            center_data(XX, y, fit_intercept=fit_intercept,
                        normalize=normalize, copy=copy,
                        sample_weight=sample_weight)

        XX = X.copy()

        X2, y2, X2_mean, X2_var, y2_mean = \
            _preprocess_data(XX, y, fit_intercept=fit_intercept,
                             normalize=normalize, copy=copy,
                             sample_weight=sample_weight)

        assert_array_almost_equal(X1, X2)
        assert_array_almost_equal(y1, y2)
        assert_array_almost_equal(X1_mean, X2_mean)
        assert_array_almost_equal(X1_var, X2_var)
        assert_array_almost_equal(y1_mean, y2_mean)

    # Sparse cases
    X = sparse.csr_matrix(X)

    for (fit_intercept, normalize, copy, sample_weight) in param_grid:

        X1, y1, X1_mean, X1_var, y1_mean = \
            center_data(X, y, fit_intercept=fit_intercept, normalize=normalize,
                        copy=copy, sample_weight=sample_weight)

        X2, y2, X2_mean, X2_var, y2_mean = \
            _preprocess_data(X, y, fit_intercept=fit_intercept,
                             normalize=normalize, copy=copy,
                             sample_weight=sample_weight, return_mean=False)

        assert_array_almost_equal(X1.toarray(), X2.toarray())
        assert_array_almost_equal(y1, y2)
        assert_array_almost_equal(X1_mean, X2_mean)
        assert_array_almost_equal(X1_var, X2_var)
        assert_array_almost_equal(y1_mean, y2_mean)

    for (fit_intercept, normalize) in product([True, False], [True, False]):

        X1, y1, X1_mean, X1_var, y1_mean = \
            sparse_center_data(X, y, fit_intercept=fit_intercept,
                               normalize=normalize)

        X2, y2, X2_mean, X2_var, y2_mean = \
            _preprocess_data(X, y, fit_intercept=fit_intercept,
                             normalize=normalize, return_mean=True)

        assert_array_almost_equal(X1.toarray(), X2.toarray())
        assert_array_almost_equal(y1, y2)
        assert_array_almost_equal(X1_mean, X2_mean)
        assert_array_almost_equal(X1_var, X2_var)
        assert_array_almost_equal(y1_mean, y2_mean)