from itertools import product
from nose.tools import assert_true

import numpy as np
from numpy.testing import assert_almost_equal, assert_array_almost_equal
from scipy import linalg

from sklearn import neighbors, manifold
from sklearn.manifold.locally_linear import barycenter_kneighbors_graph
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import ignore_warnings
from sklearn.utils.testing import assert_raise_message

eigen_solvers = ['dense', 'arpack']

#----------------------------------------------------------------------
# Test utility routines
def test_barycenter_kneighbors_graph():
    X = np.array([[0, 1], [1.01, 1.], [2, 0]])

    A = barycenter_kneighbors_graph(X, 1)
    assert_array_almost_equal(
        A.toarray(),
        [[0.,  1.,  0.],
         [1.,  0.,  0.],
         [0.,  1.,  0.]])

    A = barycenter_kneighbors_graph(X, 2)
    # check that columns sum to one
    assert_array_almost_equal(np.sum(A.toarray(), 1), np.ones(3))
    pred = np.dot(A.toarray(), X)
    assert_less(linalg.norm(pred - X) / X.shape[0], 1)

#----------------------------------------------------------------------
# Test LLE by computing the reconstruction error on some manifolds.

def test_lle_simple_grid():
    # note: ARPACK is numerically unstable, so this test will fail for
    #       some random seeds.  We choose 2 because the tests pass.
    rng = np.random.RandomState(2)

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(5), repeat=2)))
    X = X + 1e-10 * rng.uniform(size=X.shape)
    n_components = 2
    clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
                                          n_components=n_components,
                                          random_state=rng)
    tol = 0.1

    N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
    reconstruction_error = linalg.norm(np.dot(N, X) - X, 'fro')
    assert_less(reconstruction_error, tol)

    for solver in eigen_solvers:
        clf.set_params(eigen_solver=solver)
        clf.fit(X)
        assert_true(clf.embedding_.shape[1] == n_components)
        reconstruction_error = linalg.norm(
            np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2

        assert_less(reconstruction_error, tol)
        assert_almost_equal(clf.reconstruction_error_,
                            reconstruction_error, decimal=1)

    # re-embed a noisy version of X using the transform method
    noise = rng.randn(*X.shape) / 100
    X_reembedded = clf.transform(X + noise)
    assert_less(linalg.norm(X_reembedded - clf.embedding_), tol)

def test_lle_manifold():
    rng = np.random.RandomState(0)
    # similar test on a slightly more complex manifold
    X = np.array(list(product(np.arange(18), repeat=2)))
    X = np.c_[X, X[:, 0] ** 2 / 18]
    X = X + 1e-10 * rng.uniform(size=X.shape)
    n_components = 2
    for method in ["standard", "hessian", "modified", "ltsa"]:
        clf = manifold.LocallyLinearEmbedding(n_neighbors=6,
                                              n_components=n_components,
                                              method=method, random_state=0)
        tol = 1.5 if method == "standard" else 3

        N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
        reconstruction_error = linalg.norm(np.dot(N, X) - X)
        assert_less(reconstruction_error, tol)

        for solver in eigen_solvers:
            clf.set_params(eigen_solver=solver)
            clf.fit(X)
            assert_true(clf.embedding_.shape[1] == n_components)
            reconstruction_error = linalg.norm(
                np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
            details = ("solver: %s, method: %s" % (solver, method))
            assert_less(reconstruction_error, tol, msg=details)
            assert_less(np.abs(clf.reconstruction_error_ -
                               reconstruction_error),
                        tol * reconstruction_error, msg=details)

# Test the error raised when parameter passed to lle is invalid
def test_lle_init_parameters():
    X = np.random.rand(5, 3)

    clf = manifold.LocallyLinearEmbedding(eigen_solver="error")
    msg = "unrecognized eigen_solver 'error'"
    assert_raise_message(ValueError, msg, clf.fit, X)

    clf = manifold.LocallyLinearEmbedding(method="error")
    msg = "unrecognized method 'error'"
    assert_raise_message(ValueError, msg, clf.fit, X)

def test_pipeline():
    # check that LocallyLinearEmbedding works fine as a Pipeline
    # only checks that no error is raised.
    # TODO check that it actually does something useful
    from sklearn import pipeline, datasets
    X, y = datasets.make_blobs(random_state=0)
    clf = pipeline.Pipeline(
        [('filter', manifold.LocallyLinearEmbedding(random_state=0)),
         ('clf', neighbors.KNeighborsClassifier())])
    clf.fit(X, y)
    assert_less(.9, clf.score(X, y))

# Test the error raised when the weight matrix is singular
def test_singular_matrix():
    from nose.tools import assert_raises
    M = np.ones((10, 3))
    f = ignore_warnings
    assert_raises(ValueError, f(manifold.locally_linear_embedding),
                  M, 2, 1, method='standard', eigen_solver='arpack')


"""
=================================
Gaussian Mixture Model Selection
=================================

This example shows that model selection can be performed with
Gaussian Mixture Models using information-theoretic criteria (BIC).
Model selection concerns both the covariance type
and the number of components in the model.
In that case, AIC also provides the right result (not shown to save time),
but BIC is better suited if the problem is to identify the right model.
Unlike Bayesian procedures, such inferences are prior-free.

In that case, the model with 2 components and full covariance
(which corresponds to the true generative model) is selected.
"""
print(__doc__)

import itertools

import numpy as np
from scipy import linalg
import matplotlib.pyplot as plt
import matplotlib as mpl

from sklearn import mixture

# Number of samples per component
n_samples = 500

# Generate random sample, two components
np.random.seed(0)
C = np.array([[0., -0.1], [1.7, .4]])
X = np.r_[np.dot(np.random.randn(n_samples, 2), C),
          .7 * np.random.randn(n_samples, 2) + np.array([-6, 3])]

lowest_bic = np.infty
bic = []
n_components_range = range(1, 7)
cv_types = ['spherical', 'tied', 'diag', 'full']
for cv_type in cv_types:
    for n_components in n_components_range:
        # Fit a mixture of Gaussians with EM
        gmm = mixture.GMM(n_components=n_components, covariance_type=cv_type)
        gmm.fit(X)
        bic.append(gmm.bic(X))
        if bic[-1] < lowest_bic:
            lowest_bic = bic[-1]
            best_gmm = gmm

bic = np.array(bic)
color_iter = itertools.cycle(['k', 'r', 'g', 'b', 'c', 'm', 'y'])
clf = best_gmm
bars = []

# Plot the BIC scores
spl = plt.subplot(2, 1, 1)
for i, (cv_type, color) in enumerate(zip(cv_types, color_iter)):
    xpos = np.array(n_components_range) + .2 * (i - 2)
    bars.append(plt.bar(xpos, bic[i * len(n_components_range):
                                  (i + 1) * len(n_components_range)],
                        width=.2, color=color))
plt.xticks(n_components_range)
plt.ylim([bic.min() * 1.01 - .01 * bic.max(), bic.max()])
plt.title('BIC score per model')
xpos = np.mod(bic.argmin(), len(n_components_range)) + .65 +\
    .2 * np.floor(bic.argmin() / len(n_components_range))
plt.text(xpos, bic.min() * 0.97 + .03 * bic.max(), '*', fontsize=14)
spl.set_xlabel('Number of components')
spl.legend([b[0] for b in bars], cv_types)

# Plot the winner
splot = plt.subplot(2, 1, 2)
Y_ = clf.predict(X)
for i, (mean, covar, color) in enumerate(zip(clf.means_, clf.covars_,
                                             color_iter)):
    v, w = linalg.eigh(covar)
    if not np.any(Y_ == i):
        continue
    plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], .8, color=color)

    # Plot an ellipse to show the Gaussian component
    angle = np.arctan2(w[0][1], w[0][0])
    angle = 180 * angle / np.pi  # convert to degrees
    v *= 4
    ell = mpl.patches.Ellipse(mean, v[0], v[1], 180 + angle, color=color)
    ell.set_clip_box(splot.bbox)
    ell.set_alpha(.5)
    splot.add_artist(ell)

plt.xlim(-10, 10)
plt.ylim(-3, 6)
plt.xticks(())
plt.yticks(())
plt.title('Selected GMM: full model, 2 components')
plt.subplots_adjust(hspace=.35, bottom=.02)
plt.show()


"""
A comparison of different methods in GLM

Data comes from a random square matrix.

"""
from datetime import datetime
import numpy as np
from sklearn import linear_model
from sklearn.utils.bench import total_seconds

if __name__ == '__main__':

    import pylab as pl

    n_iter = 40

    time_ridge = np.empty(n_iter)
    time_ols = np.empty(n_iter)
    time_lasso = np.empty(n_iter)

    dimensions = 500 * np.arange(1, n_iter + 1)

    for i in range(n_iter):

        print('Iteration %s of %s' % (i, n_iter))

        n_samples, n_features = 10 * i + 3, 10 * i + 3

        X = np.random.randn(n_samples, n_features)
        Y = np.random.randn(n_samples)

        start = datetime.now()
        ridge = linear_model.Ridge(alpha=1.)
        ridge.fit(X, Y)
        time_ridge[i] = total_seconds(datetime.now() - start)

        start = datetime.now()
        ols = linear_model.LinearRegression()
        ols.fit(X, Y)
        time_ols[i] = total_seconds(datetime.now() - start)

        start = datetime.now()
        lasso = linear_model.LassoLars()
        lasso.fit(X, Y)
        time_lasso[i] = total_seconds(datetime.now() - start)

    pl.figure('scikit-learn GLM benchmark results')
    pl.xlabel('Dimensions')
    pl.ylabel('Time (s)')
    pl.plot(dimensions, time_ridge, color='r')
    pl.plot(dimensions, time_ols, color='g')
    pl.plot(dimensions, time_lasso, color='b')

    pl.legend(['Ridge', 'OLS', 'LassoLars'], loc='upper left')
    pl.axis('tight')
    pl.show()


# -*- coding: utf-8 -*-
"""
Authors: Tim Hessels
         UNESCO-IHE 2016
Contact: t.hessels@unesco-ihe.org
Repository: https://github.com/wateraccounting/wa
Module: Collect/CFSR
"""
# General modules
import pandas as pd
import os
import numpy as np
from netCDF4 import Dataset
import re
from joblib import Parallel, delayed

# WA+ modules
from wa.Collect.CFSR.Download_data_CFSR import Download_data
from wa.General import data_conversions as DC

def CollectData(Dir, Var, Startdate, Enddate, latlim, lonlim, Waitbar, cores, Version):
    """
    This function collects daily CFSR data in geotiff format

    Keyword arguments:
    Dir -- 'C:/file/to/path/'
    Var -- 'dlwsfc','dswsfc','ulwsfc', or 'uswsfc'
    Startdate -- 'yyyy-mm-dd'
    Enddate -- 'yyyy-mm-dd'
    latlim -- [ymin, ymax] (values must be between -50 and 50)
    lonlim -- [xmin, xmax] (values must be between -180 and 180)
    Waitbar -- 1 (Default) will print a wait bar
    cores -- The number of cores used to run the routine.
             It can be 'False' to avoid using parallel computing
		    routines.
    Version -- 1 or 2 (1 = CFSR, 2 = CFSRv2)
    """

    # Creates an array of the days of which the ET is taken
    Dates = pd.date_range(Startdate,Enddate,freq = 'D')

    # Create Waitbar
    if Waitbar == 1:
        import wa.Functions.Start.WaitbarConsole as WaitbarConsole
        total_amount = len(Dates)
        amount = 0
        WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)

    # For collecting CFSR data
    if Version == 1:
        # Check the latitude and longitude and otherwise set lat or lon on greatest extent
        if latlim[0] < -89.9171038899 or latlim[1] > 89.9171038899:
            print 'Latitude above 89.917N or below 89.917S is not possible. Value set to maximum'
            latlim[0] = np.maximum(latlim[0],-89.9171038899)
            latlim[1] = np.minimum(latlim[1],89.9171038899)
        if lonlim[0] < -180 or lonlim[1] > 179.843249782:
            print 'Longitude must be between 179.84E and 179.84W. Now value is set to maximum'
            lonlim[0] = np.maximum(lonlim[0],-180)
            lonlim[1] = np.minimum(lonlim[1],179.843249782)

        # Make directory for the CFSR data
        output_folder=os.path.join(Dir,'Radiation','CFSR')
        if not os.path.exists(output_folder):
            os.makedirs(output_folder)

    # For collecting CFSRv2 data
    if Version == 2:
            # Check the latitude and longitude and otherwise set lat or lon on greatest extent
        if latlim[0] < -89.9462116040955806 or latlim[1] > 89.9462116040955806:
            print 'Latitude above 89.917N or below 89.946S is not possible. Value set to maximum'
            latlim[0] = np.maximum(latlim[0],-89.9462116040955806)
            latlim[1] = np.minimum(latlim[1],89.9462116040955806)
        if lonlim[0] < -180 or lonlim[1] > 179.8977275:
            print 'Longitude must be between 179.90E and 179.90W. Now value is set to maximum'
            lonlim[0] = np.maximum(lonlim[0],-180)
            lonlim[1] = np.minimum(lonlim[1],179.8977275)

        # Make directory for the CFSRv2 data
        output_folder=os.path.join(Dir,'Radiation','CFSRv2')
        if not os.path.exists(output_folder):
            os.makedirs(output_folder)

    # Pass variables to parallel function and run
    args = [output_folder, latlim, lonlim, Var, Version]
    if not cores:
        for Date in Dates:
            RetrieveData(Date, args)
            if Waitbar == 1:
                amount += 1
                WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
        results = True
    else:
        results = Parallel(n_jobs=cores)(delayed(RetrieveData)(Date, args)
                                         for Date in Dates)

    # Remove all .nc and .grb2 files
    for f in os.listdir(output_folder):
        if re.search(".nc", f):
            os.remove(os.path.join(output_folder, f))
    for f in os.listdir(output_folder):
        if re.search(".grb2", f):
            os.remove(os.path.join(output_folder, f))
    for f in os.listdir(output_folder):
        if re.search(".grib2", f):
            os.remove(os.path.join(output_folder, f))

    return results

def RetrieveData(Date, args):

    # unpack the arguments
    [output_folder, latlim, lonlim, Var, Version] = args

    # Name of the model
    if Version == 1:
        version_name = 'CFSR'
    if Version == 2:
        version_name = 'CFSRv2'

    # Name of the outputfile
    if Var == 'dlwsfc':
        Outputname = 'DLWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'
    if Var == 'dswsfc':
        Outputname = 'DSWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'
    if Var == 'ulwsfc':
        Outputname = 'ULWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'
    if Var == 'uswsfc':
        Outputname = 'USWR_%s_W-m2_' %version_name + str(Date.strftime('%Y')) + '.' + str(Date.strftime('%m')) + '.' + str(Date.strftime('%d')) + '.tif'

    # Create the total end output name
    outputnamePath = os.path.join(output_folder, Outputname)

    # If the output name not exists than create this output
    if not os.path.exists(outputnamePath):

        local_filename = Download_data(Date, Version, output_folder, Var)

        # convert grb2 to netcdf (wgrib2 module is needed)
        for i in range(0,4):
            nameNC = 'Output' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + str(Date.strftime('%d')) + '-' + str(i+1) + '.nc'

            # Total path of the output
            FileNC6hour = os.path.join(output_folder, nameNC)

	       # Band number of the grib data which is converted in .nc
            band=(int(Date.strftime('%d')) - 1) * 28 + (i + 1) * 7

            # Convert the data
            DC.Convert_grb2_to_nc(local_filename, FileNC6hour, band)

        if Version == 1:

            if Date < pd.Timestamp(pd.datetime(2011, 01, 01)):

                # Convert the latlim and lonlim into array
                Xstart = np.floor((lonlim[0] + 180.1562497) / 0.3125)
                Xend = np.ceil((lonlim[1] + 180.1562497) / 0.3125) + 1
                Ystart = np.floor((latlim[0] + 89.9171038899) / 0.3122121663)
                Yend = np.ceil((latlim[1] + 89.9171038899) / 0.3122121663)

                # Create a new dataset
                Datatot = np.zeros([576, 1152])

            else:
                Version = 2

        if Version == 2:

            # Convert the latlim and lonlim into array
            Xstart = np.floor((lonlim[0] + 180.102272725) / 0.204545)
            Xend = np.ceil((lonlim[1] + 180.102272725) / 0.204545) + 1
            Ystart = np.floor((latlim[0] + 89.9462116040955806) / 0.204423)
            Yend = np.ceil((latlim[1] + 89.9462116040955806) / 0.204423)

            # Create a new dataset
            Datatot = np.zeros([880, 1760])

        # Open 4 times 6 hourly dataset
        for i in range (0, 4):
            nameNC = 'Output' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + str(Date.strftime('%d')) + '-' + str(i + 1) + '.nc'
            FileNC6hour = os.path.join(output_folder, nameNC)
            f = Dataset(FileNC6hour, mode = 'r')
            Data = f.variables['Band1'][0:int(Datatot.shape[0]), 0:int(Datatot.shape[1])]
            f.close()
            data = np.array(Data)
            Datatot = Datatot + data

        # Calculate the average in W/m^2 over the day
        DatatotDay = Datatot / 4
        DatatotDayEnd = np.zeros([int(Datatot.shape[0]), int(Datatot.shape[1])])
        DatatotDayEnd[:,0:int(Datatot.shape[0])] = DatatotDay[:, int(Datatot.shape[0]):int(Datatot.shape[1])]
        DatatotDayEnd[:,int(Datatot.shape[0]):int(Datatot.shape[1])] = DatatotDay[:, 0:int(Datatot.shape[0])]

        # clip the data to the extent difined by the user
        DatasetEnd = DatatotDayEnd[int(Ystart):int(Yend), int(Xstart):int(Xend)]

        # save file
        if Version == 1:
	        pixel_size = 0.3125
        if Version == 2:
	        pixel_size = 0.204545
        geo = [lonlim[0],pixel_size,0,latlim[1],0,-pixel_size]
        DC.Save_as_tiff(data = np.flipud(DatasetEnd), name = outputnamePath, geo = geo, projection = "WGS84")

    return()


# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for ConstantOp."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.eager import context
from tensorflow.python.eager import test
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes as dtypes_lib
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.util import compat

# TODO(josh11b): add tests with lists/tuples, Shape.
# TODO(ashankar): Collapse with tests in constant_op_test.py and use something
# like the test_util.run_in_graph_and_eager_modes decorator to confirm
# equivalence between graph and eager execution.
class ConstantTest(test.TestCase):

  def _testCpu(self, x):
    np_ans = np.array(x)
    with context.device("/device:CPU:0"):
      tf_ans = ops.convert_to_tensor(x).numpy()
    if np_ans.dtype in [np.float32, np.float64, np.complex64, np.complex128]:
      self.assertAllClose(np_ans, tf_ans)
    else:
      self.assertAllEqual(np_ans, tf_ans)

  def _testGpu(self, x):
    device = test_util.gpu_device_name()
    if device:
      np_ans = np.array(x)
      with context.device(device):
        tf_ans = ops.convert_to_tensor(x).numpy()
      if np_ans.dtype in [np.float32, np.float64, np.complex64, np.complex128]:
        self.assertAllClose(np_ans, tf_ans)
      else:
        self.assertAllEqual(np_ans, tf_ans)

  def _testAll(self, x):
    self._testCpu(x)
    self._testGpu(x)

  def testFloat(self):
    self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float32))
    self._testAll(
        np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float32))
    self._testAll(np.empty((2, 0, 5)).astype(np.float32))

    orig = [-1.0, 2.0, 0.0]
    tf_ans = constant_op.constant(orig)
    self.assertEqual(dtypes_lib.float32, tf_ans.dtype)
    self.assertAllClose(np.array(orig), tf_ans.numpy())

    # Mix floats and ints
    orig = [-1.5, 2, 0]
    tf_ans = constant_op.constant(orig)
    self.assertEqual(dtypes_lib.float32, tf_ans.dtype)
    self.assertAllClose(np.array(orig), tf_ans.numpy())

    orig = [-5, 2.5, 0]
    tf_ans = constant_op.constant(orig)
    self.assertEqual(dtypes_lib.float32, tf_ans.dtype)
    self.assertAllClose(np.array(orig), tf_ans.numpy())

    # Mix floats and ints that don't fit in int32
    orig = [1, 2**42, 0.5]
    tf_ans = constant_op.constant(orig)
    self.assertEqual(dtypes_lib.float32, tf_ans.dtype)
    self.assertAllClose(np.array(orig), tf_ans.numpy())

  def testDouble(self):
    self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float64))
    self._testAll(
        np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float64))
    self._testAll(np.empty((2, 0, 5)).astype(np.float64))

    orig = [-5, 2.5, 0]
    tf_ans = constant_op.constant(orig, dtypes_lib.float64)
    self.assertEqual(dtypes_lib.float64, tf_ans.dtype)
    self.assertAllClose(np.array(orig), tf_ans.numpy())

    # This integer is not exactly representable as a double, gets rounded.
    tf_ans = constant_op.constant(2**54 + 1, dtypes_lib.float64)
    self.assertEqual(2**54, tf_ans.numpy())

    # This integer is larger than all non-infinite numbers representable
    # by a double, raises an exception.
    with self.assertRaisesRegexp(ValueError, "out-of-range integer"):
      constant_op.constant(10**310, dtypes_lib.float64)

  def testInt32(self):
    self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.int32))
    self._testAll(
        (100 * np.random.normal(size=30)).reshape([2, 3, 5]).astype(np.int32))
    self._testAll(np.empty((2, 0, 5)).astype(np.int32))
    self._testAll([-1, 2])

  def testInt64(self):
    self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.int64))
    self._testAll(
        (100 * np.random.normal(size=30)).reshape([2, 3, 5]).astype(np.int64))
    self._testAll(np.empty((2, 0, 5)).astype(np.int64))
    # Should detect out of range for int32 and use int64 instead.
    orig = [2, 2**48, -2**48]
    tf_ans = constant_op.constant(orig)
    self.assertEqual(dtypes_lib.int64, tf_ans.dtype)
    self.assertAllClose(np.array(orig), tf_ans.numpy())

    # Out of range for an int64
    with self.assertRaisesRegexp(ValueError, "out-of-range integer"):
      constant_op.constant([2**72])

  def testComplex64(self):
    self._testAll(
        np.complex(1, 2) *
        np.arange(-15, 15).reshape([2, 3, 5]).astype(np.complex64))
    self._testAll(
        np.complex(1, 2) *
        np.random.normal(size=30).reshape([2, 3, 5]).astype(np.complex64))
    self._testAll(np.empty((2, 0, 5)).astype(np.complex64))

  def testComplex128(self):
    self._testAll(
        np.complex(1, 2) * np.arange(-15, 15).reshape([2, 3, 5
                                                      ]).astype(np.complex128))
    self._testAll(
        np.complex(1, 2) * np.random.normal(size=30).reshape(
            [2, 3, 5]).astype(np.complex128))
    self._testAll(np.empty((2, 0, 5)).astype(np.complex128))

  def testString(self):
    val = [compat.as_bytes(str(x)) for x in np.arange(-15, 15)]
    self._testCpu(np.array(val).reshape([2, 3, 5]))
    self._testCpu(np.empty((2, 0, 5)).astype(np.str_))

  def testStringWithNulls(self):
    val = ops.convert_to_tensor(b"\0\0\0\0").numpy()
    self.assertEqual(len(val), 4)
    self.assertEqual(val, b"\0\0\0\0")

    val = ops.convert_to_tensor(b"xx\0xx").numpy()
    self.assertEqual(len(val), 5)
    self.assertAllEqual(val, b"xx\0xx")

    nested = [[b"\0\0\0\0", b"xx\0xx"], [b"\0_\0_\0_\0", b"\0"]]
    val = ops.convert_to_tensor(nested).numpy()
    # NOTE(mrry): Do not use assertAllEqual, because it converts nested to a
    #   numpy array, which loses the null terminators.
    self.assertEqual(val.tolist(), nested)

  def testExplicitShapeNumPy(self):
    c = constant_op.constant(
        np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float32),
        shape=[2, 3, 5])
    self.assertEqual(c.get_shape(), [2, 3, 5])

  def testImplicitShapeNumPy(self):
    c = constant_op.constant(
        np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float32))
    self.assertEqual(c.get_shape(), [2, 3, 5])

  def testExplicitShapeList(self):
    c = constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[7])
    self.assertEqual(c.get_shape(), [7])

  def testExplicitShapeFill(self):
    c = constant_op.constant(12, shape=[7])
    self.assertEqual(c.get_shape(), [7])
    self.assertAllEqual([12, 12, 12, 12, 12, 12, 12], c.numpy())

  def testExplicitShapeReshape(self):
    c = constant_op.constant(
        np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float32),
        shape=[5, 2, 3])
    self.assertEqual(c.get_shape(), [5, 2, 3])

  def testImplicitShapeList(self):
    c = constant_op.constant([1, 2, 3, 4, 5, 6, 7])
    self.assertEqual(c.get_shape(), [7])

  def testExplicitShapeNumber(self):
    c = constant_op.constant(1, shape=[1])
    self.assertEqual(c.get_shape(), [1])

  def testImplicitShapeNumber(self):
    c = constant_op.constant(1)
    self.assertEqual(c.get_shape(), [])

  def testShapeTooBig(self):
    with self.assertRaises(TypeError):
      constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[10])

  def testShapeTooSmall(self):
    with self.assertRaises(TypeError):
      constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[5])

  def testShapeWrong(self):
    with self.assertRaisesRegexp(TypeError, None):
      constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[5])

  def testShape(self):
    self._testAll(constant_op.constant([1]).get_shape())

  def testDimension(self):
    x = constant_op.constant([1]).shape[0]
    self._testAll(x)

  def testDimensionList(self):
    x = [constant_op.constant([1]).shape[0]]
    self._testAll(x)

    # Mixing with regular integers is fine too
    self._testAll([1] + x)
    self._testAll(x + [1])

  def testDimensionTuple(self):
    x = constant_op.constant([1]).shape[0]
    self._testAll((x,))
    self._testAll((1, x))
    self._testAll((x, 1))

  def testInvalidLength(self):

    class BadList(list):

      def __init__(self):
        super(BadList, self).__init__([1, 2, 3])  # pylint: disable=invalid-length-returned

      def __len__(self):
        return -1

    with self.assertRaisesRegexp(ValueError, "should return >= 0"):
      constant_op.constant([BadList()])
    with self.assertRaisesRegexp(ValueError, "mixed types"):
      constant_op.constant([1, 2, BadList()])
    with self.assertRaisesRegexp(ValueError, "should return >= 0"):
      constant_op.constant(BadList())
    with self.assertRaisesRegexp(ValueError, "should return >= 0"):
      constant_op.constant([[BadList(), 2], 3])
    with self.assertRaisesRegexp(ValueError, "should return >= 0"):
      constant_op.constant([BadList(), [1, 2, 3]])
    with self.assertRaisesRegexp(ValueError, "should return >= 0"):
      constant_op.constant([BadList(), []])

    # TODO(allenl, josh11b): These cases should return exceptions rather than
    # working (currently shape checking only checks the first element of each
    # sequence recursively). Maybe the first one is fine, but the second one
    # silently truncating is rather bad.

    # with self.assertRaisesRegexp(ValueError, "should return >= 0"):
    #   constant_op.constant([[3, 2, 1], BadList()])
    # with self.assertRaisesRegexp(ValueError, "should return >= 0"):
    #   constant_op.constant([[], BadList()])

  def testSparseValuesRaiseErrors(self):
    with self.assertRaisesRegexp(ValueError, "non-rectangular Python sequence"):
      constant_op.constant([[1, 2], [3]], dtype=dtypes_lib.int32)

    with self.assertRaisesRegexp(ValueError, None):
      constant_op.constant([[1, 2], [3]])

    with self.assertRaisesRegexp(ValueError, None):
      constant_op.constant([[1, 2], [3], [4, 5]])

  # TODO(ashankar): This test fails with graph construction since
  # tensor_util.make_tensor_proto (invoked from constant_op.constant)
  # does not handle iterables (it relies on numpy conversion).
  # For consistency, should graph construction handle Python objects
  # that implement the sequence protocol (but not numpy conversion),
  # or should eager execution fail on such sequences?
  def testCustomSequence(self):

    # This is inspired by how many objects in pandas are implemented:
    # - They implement the Python sequence protocol
    # - But may raise a KeyError on __getitem__(self, 0)
    # See https://github.com/tensorflow/tensorflow/issues/20347
    class MySeq(object):

      def __getitem__(self, key):
        if key != 1 and key != 3:
          raise KeyError(key)
        return key

      def __len__(self):
        return 2

      def __iter__(self):
        l = list([1, 3])
        return l.__iter__()

    self.assertAllEqual([1, 3], self.evaluate(constant_op.constant(MySeq())))

class AsTensorTest(test.TestCase):

  def testAsTensorForTensorInput(self):
    t = constant_op.constant(10.0)
    x = ops.convert_to_tensor(t)
    self.assertIs(t, x)

  def testAsTensorForNonTensorInput(self):
    x = ops.convert_to_tensor(10.0)
    self.assertTrue(isinstance(x, ops.EagerTensor))

class ZerosTest(test.TestCase):

  def _Zeros(self, shape):
    ret = array_ops.zeros(shape)
    self.assertEqual(shape, ret.get_shape())
    return ret.numpy()

  def testConst(self):
    self.assertTrue(
        np.array_equal(self._Zeros([2, 3]), np.array([[0] * 3] * 2)))

  def testScalar(self):
    self.assertEqual(0, self._Zeros([]))
    self.assertEqual(0, self._Zeros(()))
    scalar = array_ops.zeros(constant_op.constant([], dtype=dtypes_lib.int32))
    self.assertEqual(0, scalar.numpy())

  def testDynamicSizes(self):
    np_ans = np.array([[0] * 3] * 2)
    # Creates a tensor of 2 x 3.
    d = array_ops.fill([2, 3], 12., name="fill")
    # Constructs a tensor of zeros of the same dimensions as "d".
    z = array_ops.zeros(array_ops.shape(d))
    out = z.numpy()
    self.assertAllEqual(np_ans, out)
    self.assertShapeEqual(np_ans, d)
    self.assertShapeEqual(np_ans, z)

  def testDtype(self):
    d = array_ops.fill([2, 3], 12., name="fill")
    self.assertEqual(d.get_shape(), [2, 3])
    # Test default type for both constant size and dynamic size
    z = array_ops.zeros([2, 3])
    self.assertEqual(z.dtype, dtypes_lib.float32)
    self.assertEqual([2, 3], z.get_shape())
    self.assertAllEqual(z.numpy(), np.zeros([2, 3]))
    z = array_ops.zeros(array_ops.shape(d))
    self.assertEqual(z.dtype, dtypes_lib.float32)
    self.assertEqual([2, 3], z.get_shape())
    self.assertAllEqual(z.numpy(), np.zeros([2, 3]))
    # Test explicit type control
    for dtype in [
        dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int32,
        dtypes_lib.uint8, dtypes_lib.int16, dtypes_lib.int8,
        dtypes_lib.complex64, dtypes_lib.complex128, dtypes_lib.int64,
        dtypes_lib.bool,
        # TODO(josh11b): Support string type here.
        # dtypes_lib.string
    ]:
      z = array_ops.zeros([2, 3], dtype=dtype)
      self.assertEqual(z.dtype, dtype)
      self.assertEqual([2, 3], z.get_shape())
      z_value = z.numpy()
      self.assertFalse(np.any(z_value))
      self.assertEqual((2, 3), z_value.shape)
      z = array_ops.zeros(array_ops.shape(d), dtype=dtype)
      self.assertEqual(z.dtype, dtype)
      self.assertEqual([2, 3], z.get_shape())
      z_value = z.numpy()
      self.assertFalse(np.any(z_value))
      self.assertEqual((2, 3), z_value.shape)

class ZerosLikeTest(test.TestCase):

  def _compareZeros(self, dtype, use_gpu):
    # Creates a tensor of non-zero values with shape 2 x 3.
    # NOTE(kearnes): The default numpy dtype associated with tf.string is
    # np.object (and can't be changed without breaking a lot things), which
    # causes a TypeError in constant_op.constant below. Here we catch the
    # special case of tf.string and set the numpy dtype appropriately.
    if dtype == dtypes_lib.string:
      numpy_dtype = np.string_
    else:
      numpy_dtype = dtype.as_numpy_dtype
    d = constant_op.constant(np.ones((2, 3), dtype=numpy_dtype), dtype=dtype)
    # Constructs a tensor of zeros of the same dimensions and type as "d".
    z_var = array_ops.zeros_like(d)
    # Test that the type is correct
    self.assertEqual(z_var.dtype, dtype)
    # Test that the shape is correct
    self.assertEqual([2, 3], z_var.get_shape())

    # Test that the value is correct
    z_value = z_var.numpy()
    self.assertFalse(np.any(z_value))
    self.assertEqual((2, 3), z_value.shape)

  def testZerosLikeCPU(self):
    for dtype in [
        dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int32,
        dtypes_lib.uint8, dtypes_lib.int16, dtypes_lib.int8,
        dtypes_lib.complex64, dtypes_lib.complex128, dtypes_lib.int64,
        # TODO(josh11b): Support string type here.
        # dtypes_lib.string
    ]:
      self._compareZeros(dtype, use_gpu=False)

  def testZerosLikeGPU(self):
    for dtype in [
        dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int32,
        dtypes_lib.bool, dtypes_lib.int64,
        # TODO(josh11b): Support string type here.
        # dtypes_lib.string
    ]:
      self._compareZeros(dtype, use_gpu=True)

  def testZerosLikeDtype(self):
    # Make sure zeros_like works even for dtypes that cannot be cast between
    shape = (3, 5)
    dtypes = np.float32, np.complex64
    for in_type in dtypes:
      x = np.arange(15).astype(in_type).reshape(*shape)
      for out_type in dtypes:
        y = array_ops.zeros_like(x, dtype=out_type).numpy()
        self.assertEqual(y.dtype, out_type)
        self.assertEqual(y.shape, shape)
        self.assertAllEqual(y, np.zeros(shape, dtype=out_type))

class OnesTest(test.TestCase):

  def _Ones(self, shape):
    ret = array_ops.ones(shape)
    self.assertEqual(shape, ret.get_shape())
    return ret.numpy()

  def testConst(self):
    self.assertTrue(np.array_equal(self._Ones([2, 3]), np.array([[1] * 3] * 2)))

  def testScalar(self):
    self.assertEqual(1, self._Ones([]))
    self.assertEqual(1, self._Ones(()))
    scalar = array_ops.ones(constant_op.constant([], dtype=dtypes_lib.int32))
    self.assertEqual(1, scalar.numpy())

  def testDynamicSizes(self):
    np_ans = np.array([[1] * 3] * 2)
    # Creates a tensor of 2 x 3.
    d = array_ops.fill([2, 3], 12., name="fill")
    # Constructs a tensor of ones of the same dimensions as "d".
    z = array_ops.ones(array_ops.shape(d))
    out = z.numpy()
    self.assertAllEqual(np_ans, out)
    self.assertShapeEqual(np_ans, d)
    self.assertShapeEqual(np_ans, z)

  def testDtype(self):
    d = array_ops.fill([2, 3], 12., name="fill")
    self.assertEqual(d.get_shape(), [2, 3])
    # Test default type for both constant size and dynamic size
    z = array_ops.ones([2, 3])
    self.assertEqual(z.dtype, dtypes_lib.float32)
    self.assertEqual([2, 3], z.get_shape())
    self.assertAllEqual(z.numpy(), np.ones([2, 3]))
    z = array_ops.ones(array_ops.shape(d))
    self.assertEqual(z.dtype, dtypes_lib.float32)
    self.assertEqual([2, 3], z.get_shape())
    self.assertAllEqual(z.numpy(), np.ones([2, 3]))
    # Test explicit type control
    for dtype in (dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int32,
                  dtypes_lib.uint8, dtypes_lib.int16, dtypes_lib.int8,
                  dtypes_lib.complex64, dtypes_lib.complex128, dtypes_lib.int64,
                  dtypes_lib.bool):
      z = array_ops.ones([2, 3], dtype=dtype)
      self.assertEqual(z.dtype, dtype)
      self.assertEqual([2, 3], z.get_shape())
      self.assertAllEqual(z.numpy(), np.ones([2, 3]))
      z = array_ops.ones(array_ops.shape(d), dtype=dtype)
      self.assertEqual(z.dtype, dtype)
      self.assertEqual([2, 3], z.get_shape())
      self.assertAllEqual(z.numpy(), np.ones([2, 3]))

class OnesLikeTest(test.TestCase):

  def testOnesLike(self):
    for dtype in [
        dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int32,
        dtypes_lib.uint8, dtypes_lib.int16, dtypes_lib.int8,
        dtypes_lib.complex64, dtypes_lib.complex128, dtypes_lib.int64
    ]:
      numpy_dtype = dtype.as_numpy_dtype
      # Creates a tensor of non-zero values with shape 2 x 3.
      d = constant_op.constant(np.ones((2, 3), dtype=numpy_dtype), dtype=dtype)
      # Constructs a tensor of zeros of the same dimensions and type as "d".
      z_var = array_ops.ones_like(d)
      # Test that the type is correct
      self.assertEqual(z_var.dtype, dtype)
      z_value = z_var.numpy()

      # Test that the value is correct
      self.assertTrue(np.array_equal(z_value, np.array([[1] * 3] * 2)))
      self.assertEqual([2, 3], z_var.get_shape())

class FillTest(test.TestCase):

  def _compare(self, dims, val, np_ans, use_gpu):
    ctx = context.context()
    device = "GPU:0" if (use_gpu and ctx.num_gpus()) else "CPU:0"
    with ops.device(device):
      tf_ans = array_ops.fill(dims, val, name="fill")
      out = tf_ans.numpy()
    self.assertAllClose(np_ans, out)

  def _compareAll(self, dims, val, np_ans):
    self._compare(dims, val, np_ans, False)
    self._compare(dims, val, np_ans, True)

  def testFillFloat(self):
    np_ans = np.array([[3.1415] * 3] * 2).astype(np.float32)
    self._compareAll([2, 3], np_ans[0][0], np_ans)

  def testFillDouble(self):
    np_ans = np.array([[3.1415] * 3] * 2).astype(np.float64)
    self._compareAll([2, 3], np_ans[0][0], np_ans)

  def testFillInt32(self):
    np_ans = np.array([[42] * 3] * 2).astype(np.int32)
    self._compareAll([2, 3], np_ans[0][0], np_ans)

  def testFillInt64(self):
    np_ans = np.array([[-42] * 3] * 2).astype(np.int64)
    self._compareAll([2, 3], np_ans[0][0], np_ans)

  def testFillComplex64(self):
    np_ans = np.array([[0.15] * 3] * 2).astype(np.complex64)
    self._compare([2, 3], np_ans[0][0], np_ans, use_gpu=False)

  def testFillComplex128(self):
    np_ans = np.array([[0.15] * 3] * 2).astype(np.complex128)
    self._compare([2, 3], np_ans[0][0], np_ans, use_gpu=False)

  def testFillString(self):
    np_ans = np.array([[b"yolo"] * 3] * 2)
    tf_ans = array_ops.fill([2, 3], np_ans[0][0], name="fill").numpy()
    self.assertAllEqual(np_ans, tf_ans)

  def testFillNegative(self):
    for shape in (-1,), (2, -1), (-1, 2), (-2), (-3):
      with self.assertRaises(errors_impl.InvalidArgumentError):
        array_ops.fill(shape, 7)

  def testShapeFunctionEdgeCases(self):
    # Non-vector dimensions.
    with self.assertRaises(errors_impl.InvalidArgumentError):
      array_ops.fill([[0, 1], [2, 3]], 1.0)

    # Non-scalar value.
    with self.assertRaises(errors_impl.InvalidArgumentError):
      array_ops.fill([3, 2], [1.0, 2.0])

if __name__ == "__main__":
  test.main()


"""Initiate file analysis.

This module is used to find the discrepancies between two given files
"""
import argparse
import csv
import logging

import pandas as pd

logger = logging.getLogger(__file__)
logging.basicConfig(level=logging.DEBUG)

mathching_records_path = 'matching_records.csv'
non_mathching_records_path = 'non_matching_records.csv'
records_diff = "customers_in_chartio_but_not_in_responsys.csv"
no_project_key_path = 'records_with_no_project_key.csv'
no_customer_key_path = 'records_with_no_customer_key.csv'
no_project_status_path = 'records_with_no_project_status.csv'

CHARTIO_GRADES = ['exceeded', 'failed', 'passed', 'ungradeable']

def get_file_df(file_path):
    """Read the file from file path then create a Pandas data frame from file.

    It eventually extracts the needed keys from this df and stored it in a dict
    Args:
        file_path(str): This is the path to the csv file to be read
    Returns:
        target_dict(dict): This holds key value pairs for future comparison
    """
    logger.info("Reading CSV file.")
    contacts_df = pd.read_csv(file_path)
    target_dict = dict()
    for contact_info in contacts_df.itertuples():
        # Each unique_key is a concatenation of the contact_info's account_key
        # and the project_id.
        _, contact_id, project_id = contact_info
        unique_key = "{x}-{y}".format(x=contact_id, y=project_id)
        target_dict.update({unique_key: contact_info})
    return target_dict

def write_to_csv(file_path, content):
    """Write content to file.

    This simple method writes the given content to the file in the  specified
    file path.
    It creates a new file in the path if no file exists there
    It keeps appending to the file per call.
    Args:
        file_path(str): Path to file
        content(list): A list of records which represent each row of the file
    TODO: Make whole process run faster by opening each file during the
    whole process
    Opening and closing a file per call slows down the whole write process
    """
    with open(file_path, 'a') as f:
        writer = csv.writer(f)
        writer.writerow(content)

def get_unique_key(project_status, project_id, customer_id):
    """Return unique key from given record.

    Returns:
        unique_key(str): This is the unique key which is used to search
        the dict which holds the overall records
    """
    project_key = project_id.replace('.0', '')
    customer_key = customer_id.replace('.0', '')
    record = [project_status, project_key, customer_key]
    invalid_result = project_status not in CHARTIO_GRADES
    invalid_project_key = project_key == 'nan'
    invalid_customer_key = customer_key == 'nan'
    if invalid_result or invalid_project_key or invalid_customer_key:
        if invalid_result and project_status == 'nan':
            record[0] = None
            write_to_csv(no_project_status_path, record)
            return False
        if project_key == 'nan':
            record[1] = None
            write_to_csv(no_project_key_path, record)
            return False
        elif customer_key == 'nan':
            record[2] = None
            write_to_csv(no_customer_key_path, record)
            return False
    else:
        unique_key = "{x}-{y}".format(x=customer_key, y=project_key)
        return unique_key

def translate_result(student_grade):
    """Interprete what a student_grade in one file means in another.

    Args:
        student_grade(str): a string which represents the grade the student
        in one file
    Returns:
        Student grade equivalent in another file
    """
    thesauraus = {
        'ungradeable': ['INCOMPLETE', 'UNGRADED', 'SUBMITTED'],
        'failed': ['INCOMPLETE'],
        'passed': ['PASSED'],
        'exceeded': ['DISTINCTION']
    }
    return thesauraus[student_grade]

def check_status(unique_key, project_status, keys_dict):
    """Compare two status against each other.

    Compares two statuses against each other and calls the appropriate function
    """
    result_list = translate_result(project_status)
    try:
        unique_record = keys_dict[unique_key]
        project_result = unique_record[3]
        if project_result in result_list:
            record = list(unique_record)[1:4]
            record.append(project_status)
            write_to_csv(mathching_records_path, record)
        else:
            record = list(unique_record)[1:4]
            record.append(project_status)
            write_to_csv(non_mathching_records_path, record)
    except (KeyError, ValueError, TypeError):
        account_project_keys = unique_key.split('-')
        record = [
            account_project_keys[0],
            account_project_keys[1],
            project_status
        ]
        write_to_csv(records_diff, record)

def compare_keys_with_files(file_path, keys_dict):
    """Go through a file and extract and processes its contents."""
    contacts_df = pd.read_csv(file_path)
    for contact_info in contacts_df.itertuples():
        index, project_status, project_key, customer_key = contact_info
        unique_key = get_unique_key(
            str(project_status),
            str(project_key),
            str(customer_key)
        )
        if unique_key:
            check_status(unique_key, str(project_status), keys_dict)

def main():
    """Run all scripts from here.

    This is the master script that initiates all the other scripts.
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('path1', help="Path to first CSV file.")
    parser.add_argument('path2', help="Path to second CSV file.")
    args = parser.parse_args()
    account_project_keys = get_file_df(args.path1)
    compare_keys_with_files(args.path2, account_project_keys)

if __name__ == '__main__':
    main()


import numpy as np
from numpy.testing import assert_array_almost_equal
from sklearn.neighbors.kd_tree import (KDTree, NeighborsHeap,
                                       simultaneous_sort, kernel_norm,
                                       nodeheap_sort, DTYPE, ITYPE)
from sklearn.neighbors.dist_metrics import DistanceMetric
from sklearn.utils.testing import SkipTest, assert_allclose

V = np.random.random((3, 3))
V = np.dot(V, V.T)

DIMENSION = 3

METRICS = {'euclidean': {},
           'manhattan': {},
           'chebyshev': {},
           'minkowski': dict(p=3)}

def brute_force_neighbors(X, Y, k, metric, **kwargs):
    D = DistanceMetric.get_metric(metric, **kwargs).pairwise(Y, X)
    ind = np.argsort(D, axis=1)[:, :k]
    dist = D[np.arange(Y.shape[0])[:, None], ind]
    return dist, ind

def test_kd_tree_query():
    np.random.seed(0)
    X = np.random.random((40, DIMENSION))
    Y = np.random.random((10, DIMENSION))

    def check_neighbors(dualtree, breadth_first, k, metric, kwargs):
        kdt = KDTree(X, leaf_size=1, metric=metric, **kwargs)
        dist1, ind1 = kdt.query(Y, k, dualtree=dualtree,
                                breadth_first=breadth_first)
        dist2, ind2 = brute_force_neighbors(X, Y, k, metric, **kwargs)

        # don't check indices here: if there are any duplicate distances,
        # the indices may not match.  Distances should not have this problem.
        assert_array_almost_equal(dist1, dist2)

    for (metric, kwargs) in METRICS.items():
        for k in (1, 3, 5):
            for dualtree in (True, False):
                for breadth_first in (True, False):
                    yield (check_neighbors,
                           dualtree, breadth_first,
                           k, metric, kwargs)

def test_kd_tree_query_radius(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    kdt = KDTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind = kdt.query_radius(query_pt, r + eps)[0]
        i = np.where(rad <= r + eps)[0]

        ind.sort()
        i.sort()

        assert_array_almost_equal(i, ind)

def test_kd_tree_query_radius_distance(n_samples=100, n_features=10):
    np.random.seed(0)
    X = 2 * np.random.random(size=(n_samples, n_features)) - 1
    query_pt = np.zeros(n_features, dtype=float)

    eps = 1E-15  # roundoff error can cause test to fail
    kdt = KDTree(X, leaf_size=5)
    rad = np.sqrt(((X - query_pt) ** 2).sum(1))

    for r in np.linspace(rad[0], rad[-1], 100):
        ind, dist = kdt.query_radius(query_pt, r + eps, return_distance=True)

        ind = ind[0]
        dist = dist[0]

        d = np.sqrt(((query_pt - X[ind]) ** 2).sum(1))

        assert_array_almost_equal(d, dist)

def compute_kernel_slow(Y, X, kernel, h):
    d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
    norm = kernel_norm(h, X.shape[1], kernel)

    if kernel == 'gaussian':
        return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
    elif kernel == 'tophat':
        return norm * (d < h).sum(-1)
    elif kernel == 'epanechnikov':
        return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
    elif kernel == 'exponential':
        return norm * (np.exp(-d / h)).sum(-1)
    elif kernel == 'linear':
        return norm * ((1 - d / h) * (d < h)).sum(-1)
    elif kernel == 'cosine':
        return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
    else:
        raise ValueError('kernel not recognized')

def test_kd_tree_kde(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    kdt = KDTree(X, leaf_size=10)

    for kernel in ['gaussian', 'tophat', 'epanechnikov',
                   'exponential', 'linear', 'cosine']:
        for h in [0.01, 0.1, 1]:
            dens_true = compute_kernel_slow(Y, X, kernel, h)

            def check_results(kernel, h, atol, rtol, breadth_first):
                dens = kdt.kernel_density(Y, h, atol=atol, rtol=rtol,
                                          kernel=kernel,
                                          breadth_first=breadth_first)
                assert_allclose(dens, dens_true, atol=atol,
                                rtol=max(rtol, 1e-7))

            for rtol in [0, 1E-5]:
                for atol in [1E-6, 1E-2]:
                    for breadth_first in (True, False):
                        yield (check_results, kernel, h, atol, rtol,
                               breadth_first)

def test_gaussian_kde(n_samples=1000):
    # Compare gaussian KDE results to scipy.stats.gaussian_kde
    from scipy.stats import gaussian_kde
    np.random.seed(0)
    x_in = np.random.normal(0, 1, n_samples)
    x_out = np.linspace(-5, 5, 30)

    for h in [0.01, 0.1, 1]:
        kdt = KDTree(x_in[:, None])
        try:
            gkde = gaussian_kde(x_in, bw_method=h / np.std(x_in))
        except TypeError:
            raise SkipTest("Old scipy, does not accept explicit bandwidth.")

        dens_kdt = kdt.kernel_density(x_out[:, None], h) / n_samples
        dens_gkde = gkde.evaluate(x_out)

        assert_array_almost_equal(dens_kdt, dens_gkde, decimal=3)

def test_kd_tree_two_point(n_samples=100, n_features=3):
    np.random.seed(0)
    X = np.random.random((n_samples, n_features))
    Y = np.random.random((n_samples, n_features))
    r = np.linspace(0, 1, 10)
    kdt = KDTree(X, leaf_size=10)

    D = DistanceMetric.get_metric("euclidean").pairwise(Y, X)
    counts_true = [(D <= ri).sum() for ri in r]

    def check_two_point(r, dualtree):
        counts = kdt.two_point_correlation(Y, r=r, dualtree=dualtree)
        assert_array_almost_equal(counts, counts_true)

    for dualtree in (True, False):
        yield check_two_point, r, dualtree

def test_kd_tree_pickle():
    import pickle
    np.random.seed(0)
    X = np.random.random((10, 3))
    kdt1 = KDTree(X, leaf_size=1)
    ind1, dist1 = kdt1.query(X)

    def check_pickle_protocol(protocol):
        s = pickle.dumps(kdt1, protocol=protocol)
        kdt2 = pickle.loads(s)
        ind2, dist2 = kdt2.query(X)
        assert_array_almost_equal(ind1, ind2)
        assert_array_almost_equal(dist1, dist2)

    for protocol in (0, 1, 2):
        yield check_pickle_protocol, protocol

def test_neighbors_heap(n_pts=5, n_nbrs=10):
    heap = NeighborsHeap(n_pts, n_nbrs)

    for row in range(n_pts):
        d_in = np.random.random(2 * n_nbrs).astype(DTYPE)
        i_in = np.arange(2 * n_nbrs, dtype=ITYPE)
        for d, i in zip(d_in, i_in):
            heap.push(row, d, i)

        ind = np.argsort(d_in)
        d_in = d_in[ind]
        i_in = i_in[ind]

        d_heap, i_heap = heap.get_arrays(sort=True)

        assert_array_almost_equal(d_in[:n_nbrs], d_heap[row])
        assert_array_almost_equal(i_in[:n_nbrs], i_heap[row])

def test_node_heap(n_nodes=50):
    vals = np.random.random(n_nodes).astype(DTYPE)

    i1 = np.argsort(vals)
    vals2, i2 = nodeheap_sort(vals)

    assert_array_almost_equal(i1, i2)
    assert_array_almost_equal(vals[i1], vals2)

def test_simultaneous_sort(n_rows=10, n_pts=201):
    dist = np.random.random((n_rows, n_pts)).astype(DTYPE)
    ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE)

    dist2 = dist.copy()
    ind2 = ind.copy()

    # simultaneous sort rows using function
    simultaneous_sort(dist, ind)

    # simultaneous sort rows using numpy
    i = np.argsort(dist2, axis=1)
    row_ind = np.arange(n_rows)[:, None]
    dist2 = dist2[row_ind, i]
    ind2 = ind2[row_ind, i]

    assert_array_almost_equal(dist, dist2)
    assert_array_almost_equal(ind, ind2)


import os
from os.path import join

import numpy
from numpy.distutils.misc_util import Configuration
from sklearn._build_utils import get_blas_info

def configuration(parent_package="", top_path=None):
    config = Configuration("manifold", parent_package, top_path)
    libraries = []
    if os.name == 'posix':
        libraries.append('m')
    config.add_extension("_utils",
                         sources=["_utils.c"],
                         include_dirs=[numpy.get_include()],
                         libraries=libraries,
                         extra_compile_args=["-O3"])
    cblas_libs, blas_info = get_blas_info()
    eca = blas_info.pop('extra_compile_args', [])
    eca.append("-O4")
    config.add_extension("_barnes_hut_tsne",
                         libraries=cblas_libs,
                         sources=["_barnes_hut_tsne.c"],
                         include_dirs=[join('..', 'src', 'cblas'),
                                       numpy.get_include(),
                                       blas_info.pop('include_dirs', [])],
                         extra_compile_args=eca, **blas_info)

    config.add_subpackage('tests')

    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration().todict())