# -*- coding: utf-8 -*-
""" GAN Example
Use a generative adversarial network (GAN) to generate digit images from a
noise distribution.
References:
    - Generative adversarial nets. I Goodfellow, J Pouget-Abadie, M Mirza,
    B Xu, D Warde-Farley, S Ozair, Y. Bengio. Advances in neural information
    processing systems, 2672-2680.
Links:
    - [GAN Paper](https://arxiv.org/pdf/1406.2661.pdf).
"""

from __future__ import division, print_function, absolute_import

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import tflearn

# Data loading and preprocessing
import tflearn.datasets.mnist as mnist
X, Y, testX, testY = mnist.load_data()

image_dim = 784 # 28*28 pixels
z_dim = 200 # Noise data points
total_samples = len(X)

# Generator
def generator(x, reuse=False):
    with tf.variable_scope('Generator', reuse=reuse):
        x = tflearn.fully_connected(x, 256, activation='relu')
        x = tflearn.fully_connected(x, image_dim, activation='sigmoid')
        return x

# Discriminator
def discriminator(x, reuse=False):
    with tf.variable_scope('Discriminator', reuse=reuse):
        x = tflearn.fully_connected(x, 256, activation='relu')
        x = tflearn.fully_connected(x, 1, activation='sigmoid')
        return x

# Build Networks
gen_input = tflearn.input_data(shape=[None, z_dim], name='input_noise')
disc_input = tflearn.input_data(shape=[None, 784], name='disc_input')

gen_sample = generator(gen_input)
disc_real = discriminator(disc_input)
disc_fake = discriminator(gen_sample, reuse=True)

# Define Loss
disc_loss = -tf.reduce_mean(tf.log(disc_real) + tf.log(1. - disc_fake))
gen_loss = -tf.reduce_mean(tf.log(disc_fake))

# Build Training Ops for both Generator and Discriminator.
# Each network optimization should only update its own variable, thus we need
# to retrieve each network variables (with get_layer_variables_by_scope) and set
# 'placeholder=None' because we do not need to feed any target.
gen_vars = tflearn.get_layer_variables_by_scope('Generator')
gen_model = tflearn.regression(gen_sample, placeholder=None, optimizer='adam',
                               loss=gen_loss, trainable_vars=gen_vars,
                               batch_size=64, name='target_gen', op_name='GEN')
disc_vars = tflearn.get_layer_variables_by_scope('Discriminator')
disc_model = tflearn.regression(disc_real, placeholder=None, optimizer='adam',
                                loss=disc_loss, trainable_vars=disc_vars,
                                batch_size=64, name='target_disc', op_name='DISC')
# Define GAN model, that output the generated images.
gan = tflearn.DNN(gen_model)

# Training
# Generate noise to feed to the generator
z = np.random.uniform(-1., 1., size=[total_samples, z_dim])
# Start training, feed both noise and real images.
gan.fit(X_inputs={gen_input: z, disc_input: X},
        Y_targets=None,
        n_epoch=100)

# Generate images from noise, using the generator network.
f, a = plt.subplots(2, 10, figsize=(10, 4))
for i in range(10):
    for j in range(2):
        # Noise input.
        z = np.random.uniform(-1., 1., size=[1, z_dim])
        # Generate image from noise. Extend to 3 channels for matplot figure.
        temp = [[ii, ii, ii] for ii in list(gan.predict([z])[0])]
        a[j][i].imshow(np.reshape(temp, (28, 28, 3)))
f.show()
plt.draw()
plt.waitforbuttonpress()


"""Test truncated SVD transformer."""

import numpy as np
import scipy.sparse as sp

from sklearn.decomposition import TruncatedSVD
from sklearn.utils import check_random_state
from sklearn.utils.testing import (assert_array_almost_equal, assert_equal,
                                   assert_raises, assert_greater,
                                   assert_array_less)

# Make an X that looks somewhat like a small tf-idf matrix.
# XXX newer versions of SciPy have scipy.sparse.rand for this.
shape = 60, 55
n_samples, n_features = shape
rng = check_random_state(42)
X = rng.randint(-100, 20, np.product(shape)).reshape(shape)
X = sp.csr_matrix(np.maximum(X, 0), dtype=np.float64)
X.data[:] = 1 + np.log(X.data)
Xdense = X.A

def test_algorithms():
    svd_a = TruncatedSVD(30, algorithm="arpack")
    svd_r = TruncatedSVD(30, algorithm="randomized", random_state=42)

    Xa = svd_a.fit_transform(X)[:, :6]
    Xr = svd_r.fit_transform(X)[:, :6]
    assert_array_almost_equal(Xa, Xr)

    comp_a = np.abs(svd_a.components_)
    comp_r = np.abs(svd_r.components_)
    # All elements are equal, but some elements are more equal than others.
    assert_array_almost_equal(comp_a[:9], comp_r[:9])
    assert_array_almost_equal(comp_a[9:], comp_r[9:], decimal=3)

def test_attributes():
    for n_components in (10, 25, 41):
        tsvd = TruncatedSVD(n_components).fit(X)
        assert_equal(tsvd.n_components, n_components)
        assert_equal(tsvd.components_.shape, (n_components, n_features))

def test_too_many_components():
    for algorithm in ["arpack", "randomized"]:
        for n_components in (n_features, n_features+1):
            tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm)
            assert_raises(ValueError, tsvd.fit, X)

def test_sparse_formats():
    for fmt in ("array", "csr", "csc", "coo", "lil"):
        Xfmt = Xdense if fmt == "dense" else getattr(X, "to" + fmt)()
        tsvd = TruncatedSVD(n_components=11)
        Xtrans = tsvd.fit_transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))
        Xtrans = tsvd.transform(Xfmt)
        assert_equal(Xtrans.shape, (n_samples, 11))

def test_inverse_transform():
    for algo in ("arpack", "randomized"):
        # We need a lot of components for the reconstruction to be "almost
        # equal" in all positions. XXX Test means or sums instead?
        tsvd = TruncatedSVD(n_components=52, random_state=42)
        Xt = tsvd.fit_transform(X)
        Xinv = tsvd.inverse_transform(Xt)
        assert_array_almost_equal(Xinv, Xdense, decimal=1)

def test_integers():
    Xint = X.astype(np.int64)
    tsvd = TruncatedSVD(n_components=6)
    Xtrans = tsvd.fit_transform(Xint)
    assert_equal(Xtrans.shape, (n_samples, tsvd.n_components))

def test_explained_variance():
    # Test sparse data
    svd_a_10_sp = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_sp = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_sp = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_sp = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_sp = svd_a_10_sp.fit_transform(X)
    X_trans_r_10_sp = svd_r_10_sp.fit_transform(X)
    X_trans_a_20_sp = svd_a_20_sp.fit_transform(X)
    X_trans_r_20_sp = svd_r_20_sp.fit_transform(X)

    # Test dense data
    svd_a_10_de = TruncatedSVD(10, algorithm="arpack")
    svd_r_10_de = TruncatedSVD(10, algorithm="randomized", random_state=42)
    svd_a_20_de = TruncatedSVD(20, algorithm="arpack")
    svd_r_20_de = TruncatedSVD(20, algorithm="randomized", random_state=42)
    X_trans_a_10_de = svd_a_10_de.fit_transform(X.toarray())
    X_trans_r_10_de = svd_r_10_de.fit_transform(X.toarray())
    X_trans_a_20_de = svd_a_20_de.fit_transform(X.toarray())
    X_trans_r_20_de = svd_r_20_de.fit_transform(X.toarray())

    # helper arrays for tests below
    svds = (svd_a_10_sp, svd_r_10_sp, svd_a_20_sp, svd_r_20_sp, svd_a_10_de,
            svd_r_10_de, svd_a_20_de, svd_r_20_de)
    svds_trans = (
        (svd_a_10_sp, X_trans_a_10_sp),
        (svd_r_10_sp, X_trans_r_10_sp),
        (svd_a_20_sp, X_trans_a_20_sp),
        (svd_r_20_sp, X_trans_r_20_sp),
        (svd_a_10_de, X_trans_a_10_de),
        (svd_r_10_de, X_trans_r_10_de),
        (svd_a_20_de, X_trans_a_20_de),
        (svd_r_20_de, X_trans_r_20_de),
    )
    svds_10_v_20 = (
        (svd_a_10_sp, svd_a_20_sp),
        (svd_r_10_sp, svd_r_20_sp),
        (svd_a_10_de, svd_a_20_de),
        (svd_r_10_de, svd_r_20_de),
    )
    svds_sparse_v_dense = (
        (svd_a_10_sp, svd_a_10_de),
        (svd_a_20_sp, svd_a_20_de),
        (svd_r_10_sp, svd_r_10_de),
        (svd_r_20_sp, svd_r_20_de),
    )

    # Assert the 1st component is equal
    for svd_10, svd_20 in svds_10_v_20:
        assert_array_almost_equal(
            svd_10.explained_variance_ratio_,
            svd_20.explained_variance_ratio_[:10],
            decimal=5,
        )

    # Assert that 20 components has higher explained variance than 10
    for svd_10, svd_20 in svds_10_v_20:
        assert_greater(
            svd_20.explained_variance_ratio_.sum(),
            svd_10.explained_variance_ratio_.sum(),
        )

    # Assert that all the values are greater than 0
    for svd in svds:
        assert_array_less(0.0, svd.explained_variance_ratio_)

    # Assert that total explained variance is less than 1
    for svd in svds:
        assert_array_less(svd.explained_variance_ratio_.sum(), 1.0)

    # Compare sparse vs. dense
    for svd_sparse, svd_dense in svds_sparse_v_dense:
        assert_array_almost_equal(svd_sparse.explained_variance_ratio_,
                svd_dense.explained_variance_ratio_)

    # Test that explained_variance is correct
    for svd, transformed in svds_trans:
        total_variance = np.var(X.toarray(), axis=0).sum()
        variances = np.var(transformed, axis=0)
        true_explained_variance_ratio = variances / total_variance

        assert_array_almost_equal(
            svd.explained_variance_ratio_,
            true_explained_variance_ratio,
        )


import numpy as np
from scipy import sparse as sp

from nose.tools import assert_raises, assert_equal
from numpy.testing import assert_array_equal

from sklearn.base import BaseEstimator
from sklearn.feature_selection.base import SelectorMixin
from sklearn.utils import check_array

class StepSelector(SelectorMixin, BaseEstimator):
    """Retain every `step` features (beginning with 0)"""
    def __init__(self, step=2):
        self.step = step

    def fit(self, X, y=None):
        X = check_array(X, 'csc')
        self.n_input_feats = X.shape[1]
        return self

    def _get_support_mask(self):
        mask = np.zeros(self.n_input_feats, dtype=bool)
        mask[::self.step] = True
        return mask

support = [True, False] * 5
support_inds = [0, 2, 4, 6, 8]
X = np.arange(20).reshape(2, 10)
Xt = np.arange(0, 20, 2).reshape(2, 5)
Xinv = X.copy()
Xinv[:, 1::2] = 0
y = [0, 1]
feature_names = list('ABCDEFGHIJ')
feature_names_t = feature_names[::2]
feature_names_inv = np.array(feature_names)
feature_names_inv[1::2] = ''

def test_transform_dense():
    sel = StepSelector()
    Xt_actual = sel.fit(X, y).transform(X)
    Xt_actual2 = StepSelector().fit_transform(X, y)
    assert_array_equal(Xt, Xt_actual)
    assert_array_equal(Xt, Xt_actual2)

    # Check dtype matches
    assert_equal(np.int32, sel.transform(X.astype(np.int32)).dtype)
    assert_equal(np.float32, sel.transform(X.astype(np.float32)).dtype)

    # Check 1d list and other dtype:
    names_t_actual = sel.transform([feature_names])
    assert_array_equal(feature_names_t, names_t_actual.ravel())

    # Check wrong shape raises error
    assert_raises(ValueError, sel.transform, np.array([[1], [2]]))

def test_transform_sparse():
    sparse = sp.csc_matrix
    sel = StepSelector()
    Xt_actual = sel.fit(sparse(X)).transform(sparse(X))
    Xt_actual2 = sel.fit_transform(sparse(X))
    assert_array_equal(Xt, Xt_actual.toarray())
    assert_array_equal(Xt, Xt_actual2.toarray())

    # Check dtype matches
    assert_equal(np.int32, sel.transform(sparse(X).astype(np.int32)).dtype)
    assert_equal(np.float32, sel.transform(sparse(X).astype(np.float32)).dtype)

    # Check wrong shape raises error
    assert_raises(ValueError, sel.transform, np.array([[1], [2]]))

def test_inverse_transform_dense():
    sel = StepSelector()
    Xinv_actual = sel.fit(X, y).inverse_transform(Xt)
    assert_array_equal(Xinv, Xinv_actual)

    # Check dtype matches
    assert_equal(np.int32,
                 sel.inverse_transform(Xt.astype(np.int32)).dtype)
    assert_equal(np.float32,
                 sel.inverse_transform(Xt.astype(np.float32)).dtype)

    # Check 1d list and other dtype:
    names_inv_actual = sel.inverse_transform([feature_names_t])
    assert_array_equal(feature_names_inv, names_inv_actual.ravel())

    # Check wrong shape raises error
    assert_raises(ValueError, sel.inverse_transform, np.array([[1], [2]]))

def test_inverse_transform_sparse():
    sparse = sp.csc_matrix
    sel = StepSelector()
    Xinv_actual = sel.fit(sparse(X)).inverse_transform(sparse(Xt))
    assert_array_equal(Xinv, Xinv_actual.toarray())

    # Check dtype matches
    assert_equal(np.int32,
                 sel.inverse_transform(sparse(Xt).astype(np.int32)).dtype)
    assert_equal(np.float32,
                 sel.inverse_transform(sparse(Xt).astype(np.float32)).dtype)

    # Check wrong shape raises error
    assert_raises(ValueError, sel.inverse_transform, np.array([[1], [2]]))

def test_get_support():
    sel = StepSelector()
    sel.fit(X, y)
    assert_array_equal(support, sel.get_support())
    assert_array_equal(support_inds, sel.get_support(indices=True))


import numpy
import time
from matplotlib import pyplot

def main (params):

    angle = params['p0']
    vel = params['v0']
    sign = params['sign']

    # Plan the trajectory if it is not planned
    T = 0
    Ta = 0
    Td = 0
    dt = params['dt']
    if not params['trajectory']:
        # Maximum acceleration and velocity values in degrees/s^2 and
        # degrees/s respectively
        amax = params['acc_limit_d']*sign*(-1)
        vmax = params['vel_limit']*sign*(-1)
        v0 = vel
        h = angle
        vlim = vmax
        # Check if the trajectory is feasible
        print "abs (amax*h) >= v0**2/2.0 = %s" % (abs (amax*h) >= v0**2/2.0)
        if abs (amax*h) >= v0**2/2.0:
            # The trajectory is feasible
            # Check if the maximum value of velocity can be reached
            if abs (h*amax) > vmax**2 - v0**2/2.0:
                # The maximum value of velocity can be reached
                Ta = (vmax - v0)/amax
                Td = vmax/amax
                term1 = abs (h/vmax)
                term2 = (vmax/(2*amax)) * (1 - (v0/vmax))**2
                term3 = (vmax/(2*amax))
                T = term1 + term2 + term3
            else:
                # The maximum value of velocity can't be reached
                vlim = ((abs (h * amax) + v0**2/2.0)**(1/2.0))*sign*(-1)
                Ta = abs ((vlim - v0)/amax)
                Td = abs (vlim/amax)
                T = Ta + Td
            # end if

            # The time has to be positive
            Ta = abs (Ta)
            Td = abs (Td)
            T = abs (T)

            print "Ta = %s, Td = %s" % (Ta, Td)

            params['trajectory'] = True

            params['T'] = T
            params['Ta'] = Ta
            params['Td'] = Td
            params['T_sign'] = sign*(-1)
            params['vv'] = vlim

            # if Ta > dt and Td > dt:
            #     params['trajectory'] = True

            #     params['T'] = T
            #     params['Ta'] = Ta
            #     params['Td'] = Td
            #     params['T_sign'] = sign*(-1)
            #     params['vv'] = vlim
            # else:
            #     Ta = 0
            #     Td = 0
            #     T = 0

            # end if

    # end if

    return

def plot (params):

    t = 0
    interval = params['dt']

    # Sign
    sign = params['T_sign']

    # Maximum values
    amax = params['acc_limit_d']*sign
    vmax = params['vel_limit']*sign

    # Buffers to store the motion
    positions = []
    vels = []
    accs = []
    # Initial values of the motion
    v0 = params['v0']
    p0 = params['p0']
    vv = params['vv']

    T = params['T']
    Ta = params['Ta']
    Td = params['Td']

    # Acceleration phase
    while t < Ta:
        # Position
        pos = p0 + v0*t + ((vv - v0)/(2*Ta))*t**2
        positions.append (pos)
        # Velocity
        vel = v0 + ((vv - v0)/(Ta))*t
        vels.append (vel)
        # Acceleration
        acc = (vv - v0)/Ta
        accs.append (acc)

        t += interval
    # end while

    # Constant velocity phase
    while t < (T - Td):
        # Position
        pos = p0 + v0*(Ta/2.0) + vv*(t-(Ta/2.0))
        positions.append (pos)
        # Velocity
        vel = vv
        vels.append (vel)
        # Acceleration
        acc = 0
        accs.append (acc)

        t += interval
    # end while

    # Deceleration phase
    while t < T:
        # Position
        pos = 0 - (vv/(2*Td))*(T-t)**2
        positions.append (pos)
        # Velocity
        vel = (vv/Td)*(T-t)
        vels.append (vel)
        # Acceleration
        acc = -(vv/Td)
        accs.append (acc)

        t += interval
    # end while

    fig = pyplot.figure (1, figsize = (20,10))

    s = fig.add_subplot (311)
    p, = s.plot(positions)
    s.grid (True)
    s.set_title ("position")

    s = fig.add_subplot (312)
    p, = s.plot(vels)
    s.grid (True)
    s.set_title ("velocity")

    s = fig.add_subplot (313)
    p, = s.plot(accs)
    s.grid (True)
    s.set_title ("acceleration")

    pyplot.show ()
    pyplot.close (1)

    return

if __name__ == "__main__":
    params = {}
    # Period
    params['dt'] = 0.015
    # Flag to indicate if it is necessary to compute the trajectory
    # (not needed here)
    params['trajectory'] = False
    # Velocity, acceleration and jerk limits in degrees/s^2
    params['vel_limit'] = 150.0
    rad_to_degrees = 180.0/numpy.pi
    radius = 0.3
    # m/s^2
    params['acc_limit'] = 7.5
    # degrees/s^2
    params['acc_limit_d'] = (params['acc_limit']*rad_to_degrees)/radius

    # # p0 = 0. Checked, trajectory unfeasible
    # # p0
    # params['p0'] = 0.0
    # # v0
    # params['v0'] = 100.0

    # p0 > 50 v0 = 0. Checked, trajectory feasible
    # p0
    params['p0'] = 80.0
    # v0
    params['v0'] = 0.0

    # # p0 > 50 v0 < limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = 80.0
    # # v0
    # params['v0'] = 50.0

    # # p0 > 50 v0 = limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = 80.0
    # # v0
    # params['v0'] = 100.0

    # # p0 > 50 v0 > limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = 80.0
    # # v0
    # params['v0'] = -150.0

    # # p0 < 50 p0 > 0 v0 = 0. Checked, trajectory feasible
    # # p0
    # params['p0'] = 20.0
    # # v0
    # params['v0'] = 0.0

    # # p0 < 50 p0 > 0 v0 < limit. REVIEW IT!!!!!!!!!
    # # p0
    # params['p0'] = 20.0
    # # v0
    # params['v0'] = 50.0

    # # p0 < 50 p0 > 0 v0 = limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = 20.0
    # # v0
    # params['v0'] = 100.0

    # # p0 < 50 p0 > 0 v0 > limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = 20.0
    # # v0
    # params['v0'] = 150.0

    # # p0 < -50 v0 = 0. Checked, trajectory feasible
    # # p0
    # params['p0'] = -80.0
    # # v0
    # params['v0'] = 0.0

    # # p0 < -50 v0 < limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = -80.0
    # # v0
    # params['v0'] = 50.0

    # # p0 < -50 v0 = limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = -80.0
    # # v0
    # params['v0'] = 100.0

    # # p0 < -50 v0 > limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = -80.0
    # # v0
    # params['v0'] = 150.0

    # # p0 > -50 p0 < 0 v0 = 0. Checked, trajectory feasible
    # # p0
    # params['p0'] = -20.0
    # # v0
    # params['v0'] = 0.0

    # # p0 > -50 p0 < 0 v0 < limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = -20.0
    # # v0
    # params['v0'] = -50.0

    # # p0 > -50 p0 < 0 v0 = limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = -20.0
    # # v0
    # params['v0'] = 100.0

    # # p0 > -50 p0 < 0 v0 > limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = -20.0
    # # v0
    # params['v0'] = 150.0

    # # p0 > -50 p0 < 0 v0 > limit. Checked, trajectory feasible
    # # p0
    # params['p0'] = -20.0
    # # v0
    # params['v0'] = 200.0

    # sign
    params['sign'] = 1
#    params['sign'] = -1

#     # p0
#    params['p0'] = 11.0962258945
# #    params['p0'] = 22.0
#     # v0
#    params['v0'] = 71.19
# #    params['v0'] = 0.0

    main(params)

    print "Trajectory performed: %s" % params['trajectory']
    if params['trajectory']:
        T = params['T']
        Ta = params['Ta']
        Td = params['Td']
        print "T = %s, Ta = %s, Td = %s" %(T, Ta, Td)

        plot (params)


"""
Test the pipeline module.
"""
import numpy as np
from scipy import sparse

from sklearn.externals.six.moves import zip
from sklearn.utils.testing import assert_raises, assert_raises_regex, assert_raise_message
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal

from sklearn.base import clone
from sklearn.pipeline import Pipeline, FeatureUnion, make_pipeline, make_union
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from sklearn.cluster import KMeans
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.decomposition import PCA, RandomizedPCA, TruncatedSVD
from sklearn.datasets import load_iris
from sklearn.preprocessing import StandardScaler
from sklearn.feature_extraction.text import CountVectorizer

JUNK_FOOD_DOCS = (
    "the pizza pizza beer copyright",
    "the pizza burger beer copyright",
    "the the pizza beer beer copyright",
    "the burger beer beer copyright",
    "the coke burger coke copyright",
    "the coke burger burger",
)

class IncorrectT(object):
    """Small class to test parameter dispatching.
    """

    def __init__(self, a=None, b=None):
        self.a = a
        self.b = b

class T(IncorrectT):

    def fit(self, X, y):
        return self

    def get_params(self, deep=False):
        return {'a': self.a, 'b': self.b}

    def set_params(self, **params):
        self.a = params['a']
        return self

class TransfT(T):

    def transform(self, X, y=None):
        return X

class FitParamT(object):
    """Mock classifier
    """

    def __init__(self):
        self.successful = False
        pass

    def fit(self, X, y, should_succeed=False):
        self.successful = should_succeed

    def predict(self, X):
        return self.successful

def test_pipeline_init():
    # Test the various init parameters of the pipeline.
    assert_raises(TypeError, Pipeline)
    # Check that we can't instantiate pipelines with objects without fit
    # method
    pipe = assert_raises(TypeError, Pipeline, [('svc', IncorrectT)])
    # Smoke test with only an estimator
    clf = T()
    pipe = Pipeline([('svc', clf)])
    assert_equal(pipe.get_params(deep=True),
                 dict(svc__a=None, svc__b=None, svc=clf,
                     **pipe.get_params(deep=False)
                     ))

    # Check that params are set
    pipe.set_params(svc__a=0.1)
    assert_equal(clf.a, 0.1)
    assert_equal(clf.b, None)
    # Smoke test the repr:
    repr(pipe)

    # Test with two objects
    clf = SVC()
    filter1 = SelectKBest(f_classif)
    pipe = Pipeline([('anova', filter1), ('svc', clf)])

    # Check that we can't use the same stage name twice
    assert_raises(ValueError, Pipeline, [('svc', SVC()), ('svc', SVC())])

    # Check that params are set
    pipe.set_params(svc__C=0.1)
    assert_equal(clf.C, 0.1)
    # Smoke test the repr:
    repr(pipe)

    # Check that params are not set when naming them wrong
    assert_raises(ValueError, pipe.set_params, anova__C=0.1)

    # Test clone
    pipe2 = clone(pipe)
    assert_false(pipe.named_steps['svc'] is pipe2.named_steps['svc'])

    # Check that apart from estimators, the parameters are the same
    params = pipe.get_params(deep=True)
    params2 = pipe2.get_params(deep=True)

    for x in pipe.get_params(deep=False):
        params.pop(x)

    for x in pipe2.get_params(deep=False):
        params2.pop(x)

    # Remove estimators that where copied
    params.pop('svc')
    params.pop('anova')
    params2.pop('svc')
    params2.pop('anova')
    assert_equal(params, params2)

def test_pipeline_methods_anova():
    # Test the various methods of the pipeline (anova).
    iris = load_iris()
    X = iris.data
    y = iris.target
    # Test with Anova + LogisticRegression
    clf = LogisticRegression()
    filter1 = SelectKBest(f_classif, k=2)
    pipe = Pipeline([('anova', filter1), ('logistic', clf)])
    pipe.fit(X, y)
    pipe.predict(X)
    pipe.predict_proba(X)
    pipe.predict_log_proba(X)
    pipe.score(X, y)

def test_pipeline_fit_params():
    # Test that the pipeline can take fit parameters
    pipe = Pipeline([('transf', TransfT()), ('clf', FitParamT())])
    pipe.fit(X=None, y=None, clf__should_succeed=True)
    # classifier should return True
    assert_true(pipe.predict(None))
    # and transformer params should not be changed
    assert_true(pipe.named_steps['transf'].a is None)
    assert_true(pipe.named_steps['transf'].b is None)

def test_pipeline_raise_set_params_error():
    # Test pipeline raises set params error message for nested models.
    pipe = Pipeline([('cls', LinearRegression())])

    # expected error message
    error_msg = ('Invalid parameter %s for estimator %s. '
                 'Check the list of available parameters '
                 'with `estimator.get_params().keys()`.')

    assert_raise_message(ValueError,
                         error_msg % ('fake', 'Pipeline'),
                         pipe.set_params,
                         fake='nope')

    # nested model check
    assert_raise_message(ValueError,
                         error_msg % ("fake", pipe),
                         pipe.set_params,
                         fake__estimator='nope')

def test_pipeline_methods_pca_svm():
    # Test the various methods of the pipeline (pca + svm).
    iris = load_iris()
    X = iris.data
    y = iris.target
    # Test with PCA + SVC
    clf = SVC(probability=True, random_state=0)
    pca = PCA(n_components='mle', whiten=True)
    pipe = Pipeline([('pca', pca), ('svc', clf)])
    pipe.fit(X, y)
    pipe.predict(X)
    pipe.predict_proba(X)
    pipe.predict_log_proba(X)
    pipe.score(X, y)

def test_pipeline_methods_preprocessing_svm():
    # Test the various methods of the pipeline (preprocessing + svm).
    iris = load_iris()
    X = iris.data
    y = iris.target
    n_samples = X.shape[0]
    n_classes = len(np.unique(y))
    scaler = StandardScaler()
    pca = RandomizedPCA(n_components=2, whiten=True)
    clf = SVC(probability=True, random_state=0)

    for preprocessing in [scaler, pca]:
        pipe = Pipeline([('preprocess', preprocessing), ('svc', clf)])
        pipe.fit(X, y)

        # check shapes of various prediction functions
        predict = pipe.predict(X)
        assert_equal(predict.shape, (n_samples,))

        proba = pipe.predict_proba(X)
        assert_equal(proba.shape, (n_samples, n_classes))

        log_proba = pipe.predict_log_proba(X)
        assert_equal(log_proba.shape, (n_samples, n_classes))

        decision_function = pipe.decision_function(X)
        assert_equal(decision_function.shape, (n_samples, n_classes))

        pipe.score(X, y)

def test_fit_predict_on_pipeline():
    # test that the fit_predict method is implemented on a pipeline
    # test that the fit_predict on pipeline yields same results as applying
    # transform and clustering steps separately
    iris = load_iris()
    scaler = StandardScaler()
    km = KMeans(random_state=0)

    # first compute the transform and clustering step separately
    scaled = scaler.fit_transform(iris.data)
    separate_pred = km.fit_predict(scaled)

    # use a pipeline to do the transform and clustering in one step
    pipe = Pipeline([('scaler', scaler), ('Kmeans', km)])
    pipeline_pred = pipe.fit_predict(iris.data)

    assert_array_almost_equal(pipeline_pred, separate_pred)

def test_fit_predict_on_pipeline_without_fit_predict():
    # tests that a pipeline does not have fit_predict method when final
    # step of pipeline does not have fit_predict defined
    scaler = StandardScaler()
    pca = PCA()
    pipe = Pipeline([('scaler', scaler), ('pca', pca)])
    assert_raises_regex(AttributeError,
                        "'PCA' object has no attribute 'fit_predict'",
                        getattr, pipe, 'fit_predict')

def test_feature_union():
    # basic sanity check for feature union
    iris = load_iris()
    X = iris.data
    X -= X.mean(axis=0)
    y = iris.target
    svd = TruncatedSVD(n_components=2, random_state=0)
    select = SelectKBest(k=1)
    fs = FeatureUnion([("svd", svd), ("select", select)])
    fs.fit(X, y)
    X_transformed = fs.transform(X)
    assert_equal(X_transformed.shape, (X.shape[0], 3))

    # check if it does the expected thing
    assert_array_almost_equal(X_transformed[:, :-1], svd.fit_transform(X))
    assert_array_equal(X_transformed[:, -1],
                       select.fit_transform(X, y).ravel())

    # test if it also works for sparse input
    # We use a different svd object to control the random_state stream
    fs = FeatureUnion([("svd", svd), ("select", select)])
    X_sp = sparse.csr_matrix(X)
    X_sp_transformed = fs.fit_transform(X_sp, y)
    assert_array_almost_equal(X_transformed, X_sp_transformed.toarray())

    # test setting parameters
    fs.set_params(select__k=2)
    assert_equal(fs.fit_transform(X, y).shape, (X.shape[0], 4))

    # test it works with transformers missing fit_transform
    fs = FeatureUnion([("mock", TransfT()), ("svd", svd), ("select", select)])
    X_transformed = fs.fit_transform(X, y)
    assert_equal(X_transformed.shape, (X.shape[0], 8))

def test_make_union():
    pca = PCA()
    mock = TransfT()
    fu = make_union(pca, mock)
    names, transformers = zip(*fu.transformer_list)
    assert_equal(names, ("pca", "transft"))
    assert_equal(transformers, (pca, mock))

def test_pipeline_transform():
    # Test whether pipeline works with a transformer at the end.
    # Also test pipeline.transform and pipeline.inverse_transform
    iris = load_iris()
    X = iris.data
    pca = PCA(n_components=2)
    pipeline = Pipeline([('pca', pca)])

    # test transform and fit_transform:
    X_trans = pipeline.fit(X).transform(X)
    X_trans2 = pipeline.fit_transform(X)
    X_trans3 = pca.fit_transform(X)
    assert_array_almost_equal(X_trans, X_trans2)
    assert_array_almost_equal(X_trans, X_trans3)

    X_back = pipeline.inverse_transform(X_trans)
    X_back2 = pca.inverse_transform(X_trans)
    assert_array_almost_equal(X_back, X_back2)

def test_pipeline_fit_transform():
    # Test whether pipeline works with a transformer missing fit_transform
    iris = load_iris()
    X = iris.data
    y = iris.target
    transft = TransfT()
    pipeline = Pipeline([('mock', transft)])

    # test fit_transform:
    X_trans = pipeline.fit_transform(X, y)
    X_trans2 = transft.fit(X, y).transform(X)
    assert_array_almost_equal(X_trans, X_trans2)

def test_make_pipeline():
    t1 = TransfT()
    t2 = TransfT()

    pipe = make_pipeline(t1, t2)
    assert_true(isinstance(pipe, Pipeline))
    assert_equal(pipe.steps[0][0], "transft-1")
    assert_equal(pipe.steps[1][0], "transft-2")

    pipe = make_pipeline(t1, t2, FitParamT())
    assert_true(isinstance(pipe, Pipeline))
    assert_equal(pipe.steps[0][0], "transft-1")
    assert_equal(pipe.steps[1][0], "transft-2")
    assert_equal(pipe.steps[2][0], "fitparamt")

def test_feature_union_weights():
    # test feature union with transformer weights
    iris = load_iris()
    X = iris.data
    y = iris.target
    pca = RandomizedPCA(n_components=2, random_state=0)
    select = SelectKBest(k=1)
    # test using fit followed by transform
    fs = FeatureUnion([("pca", pca), ("select", select)],
                      transformer_weights={"pca": 10})
    fs.fit(X, y)
    X_transformed = fs.transform(X)
    # test using fit_transform
    fs = FeatureUnion([("pca", pca), ("select", select)],
                      transformer_weights={"pca": 10})
    X_fit_transformed = fs.fit_transform(X, y)
    # test it works with transformers missing fit_transform
    fs = FeatureUnion([("mock", TransfT()), ("pca", pca), ("select", select)],
                      transformer_weights={"mock": 10})
    X_fit_transformed_wo_method = fs.fit_transform(X, y)
    # check against expected result

    # We use a different pca object to control the random_state stream
    assert_array_almost_equal(X_transformed[:, :-1], 10 * pca.fit_transform(X))
    assert_array_equal(X_transformed[:, -1],
                       select.fit_transform(X, y).ravel())
    assert_array_almost_equal(X_fit_transformed[:, :-1],
                              10 * pca.fit_transform(X))
    assert_array_equal(X_fit_transformed[:, -1],
                       select.fit_transform(X, y).ravel())
    assert_equal(X_fit_transformed_wo_method.shape, (X.shape[0], 7))

def test_feature_union_parallel():
    # test that n_jobs work for FeatureUnion
    X = JUNK_FOOD_DOCS

    fs = FeatureUnion([
        ("words", CountVectorizer(analyzer='word')),
        ("chars", CountVectorizer(analyzer='char')),
    ])

    fs_parallel = FeatureUnion([
        ("words", CountVectorizer(analyzer='word')),
        ("chars", CountVectorizer(analyzer='char')),
    ], n_jobs=2)

    fs_parallel2 = FeatureUnion([
        ("words", CountVectorizer(analyzer='word')),
        ("chars", CountVectorizer(analyzer='char')),
    ], n_jobs=2)

    fs.fit(X)
    X_transformed = fs.transform(X)
    assert_equal(X_transformed.shape[0], len(X))

    fs_parallel.fit(X)
    X_transformed_parallel = fs_parallel.transform(X)
    assert_equal(X_transformed.shape, X_transformed_parallel.shape)
    assert_array_equal(
        X_transformed.toarray(),
        X_transformed_parallel.toarray()
    )

    # fit_transform should behave the same
    X_transformed_parallel2 = fs_parallel2.fit_transform(X)
    assert_array_equal(
        X_transformed.toarray(),
        X_transformed_parallel2.toarray()
    )

    # transformers should stay fit after fit_transform
    X_transformed_parallel2 = fs_parallel2.transform(X)
    assert_array_equal(
        X_transformed.toarray(),
        X_transformed_parallel2.toarray()
    )

def test_feature_union_feature_names():
    word_vect = CountVectorizer(analyzer="word")
    char_vect = CountVectorizer(analyzer="char_wb", ngram_range=(3, 3))
    ft = FeatureUnion([("chars", char_vect), ("words", word_vect)])
    ft.fit(JUNK_FOOD_DOCS)
    feature_names = ft.get_feature_names()
    for feat in feature_names:
        assert_true("chars__" in feat or "words__" in feat)
    assert_equal(len(feature_names), 35)

def test_classes_property():
    iris = load_iris()
    X = iris.data
    y = iris.target

    reg = make_pipeline(SelectKBest(k=1), LinearRegression())
    reg.fit(X, y)
    assert_raises(AttributeError, getattr, reg, "classes_")

    clf = make_pipeline(SelectKBest(k=1), LogisticRegression(random_state=0))
    assert_raises(AttributeError, getattr, clf, "classes_")
    clf.fit(X, y)
    assert_array_equal(clf.classes_, np.unique(y))


##############################################################################
# Copyright 2017-2018 Rigetti Computing
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
##############################################################################

import logging

import numpy as np
import matplotlib.pyplot as plt
from pyquil.quilbase import Pragma
from scipy.sparse import csr_matrix, coo_matrix
from pyquil.quil import Program

import grove.tomography.operator_utils
from grove.tomography.tomography import TomographyBase, TomographySettings, DEFAULT_SOLVER_KWARGS
from grove.tomography import tomography
import grove.tomography.utils as ut
import grove.tomography.operator_utils as o_ut

_log = logging.getLogger(__name__)

qt = ut.import_qutip()
cvxpy = ut.import_cvxpy()

UNIT_TRACE = 'unit_trace'
POSITIVE = 'positive'
DEFAULT_STATE_TOMO_SETTINGS = TomographySettings(
    constraints={UNIT_TRACE},
    solver_kwargs=DEFAULT_SOLVER_KWARGS
)

def _prepare_c_jk_m(readout_povm, pauli_basis, channel_ops):
    """
    Prepare the coefficient matrix for state tomography. This function uses sparse matrices
    for much greater efficiency.
    The coefficient matrix is defined as:

    .. math::

            C_{(jk)m} = \tr{\Pi_{s_j} \Lambda_k(P_m)} = \sum_{r}\pi_{jr}(\mathcal{R}_{k})_{rm}

    where :math:`\Lambda_k(\cdot)` is the quantum map corresponding to the k-th pre-measurement
    channel, i.e., :math:`\Lambda_k(\rho) = E_k \rho E_k^\dagger` where :math:`E_k` is the k-th
    channel operator. This map can also be represented via its transfer matrix
    :math:`\mathcal{R}_{k}`. In that case one also requires the overlap between the (generalized)
    Pauli basis ops and the projection operators
    :math:`\pi_{jl}:=\sbraket{\Pi_j}{P_l} = \tr{\Pi_j P_l}`.
    See the grove documentation on tomography for detailed information.

    :param DiagonalPOVM readout_povm: The POVM corresponding to the readout plus classifier.
    :param OperatorBasis pauli_basis: The (generalized) Pauli basis employed in the estimation.
    :param list channel_ops: The pre-measurement channel operators as `qutip.Qobj`
    :return: The coefficient matrix necessary to set up the binomial state tomography problem.
    :rtype: scipy.sparse.csr_matrix
    """
    channel_transfer_matrices = [pauli_basis.transfer_matrix(qt.to_super(ek)) for ek in channel_ops]

    # This bit could be more efficient but does not run super long and is thus preserved for
    #  readability.
    pi_jr = csr_matrix(
        [pauli_basis.project_op(n_j).toarray().ravel()
         for n_j in readout_povm.ops])

    # Dict used for constructing our sparse matrix, keys are tuples (row_index, col_index), values
    # are the non-zero elements of the final matrix.
    c_jk_m_elms = {}

    # This explicitly exploits the sparsity of all operators involved
    for k in range(len(channel_ops)):
        pi_jr__rk_rm = (pi_jr * channel_transfer_matrices[k]).tocoo()
        for (j, m, val) in ut.izip(pi_jr__rk_rm.row, pi_jr__rk_rm.col, pi_jr__rk_rm.data):
            # The multi-index (j,k) is enumerated in column-major ordering (like Fortran arrays)
            c_jk_m_elms[(j + k * readout_povm.pi_basis.dim, m)] = val.real

    # create sparse matrix from COO-format (see scipy.sparse docs)
    _keys, _values = ut.izip(*c_jk_m_elms.items())
    _rows, _cols = ut.izip(*_keys)
    c_jk_m = coo_matrix((list(_values), (list(_rows), list(_cols))),
                        shape=(readout_povm.pi_basis.dim * len(channel_ops),
                               pauli_basis.dim)).tocsr()
    return c_jk_m

class StateTomography(TomographyBase):
    """
    A StateTomography object encapsulates the result of quantum state estimation from tomographic
    data. It provides convenience functions for visualization and computing state fidelities.
    """
    __tomography_type__ = "STATE"

    @staticmethod
    def estimate_from_ssr(histograms, readout_povm, channel_ops, settings):
        """
        Estimate a density matrix from single shot histograms obtained by measuring bitstrings in
        the Z-eigenbasis after application of given channel operators.

        :param numpy.ndarray histograms: The single shot histograms, `shape=(n_channels, dim)`.
        :param DiagognalPOVM readout_povm: The POVM corresponding to the readout plus classifier.
        :param list channel_ops: The tomography measurement channels as `qutip.Qobj`'s.
        :param TomographySettings settings: The solver and estimation settings.
        :return: The generated StateTomography object.
        :rtype: StateTomography
        """
        nqc = len(channel_ops[0].dims[0])
        pauli_basis = grove.tomography.operator_utils.PAULI_BASIS ** nqc
        pi_basis = readout_povm.pi_basis

        if not histograms.shape[1] == pi_basis.dim:  # pragma no coverage
            raise ValueError("Currently tomography is only implemented for two-level systems.")

        # prepare the log-likelihood function parameters, see documentation
        n_kj = np.asarray(histograms)
        c_jk_m = _prepare_c_jk_m(readout_povm, pauli_basis, channel_ops)
        rho_m = cvxpy.Variable(pauli_basis.dim)

        p_jk = c_jk_m * rho_m
        obj = -n_kj.ravel() * cvxpy.log(p_jk)

        p_jk_mat = cvxpy.reshape(p_jk, pi_basis.dim, len(channel_ops))  # cvxpy has col-major order

        # Default constraints:
        # MLE must describe valid probability distribution
        # i.e., for each k, p_jk must sum to one and be element-wise non-negative:
        # 1. \sum_j p_jk == 1  for all k
        # 2. p_jk >= 0         for all j, k
        # where p_jk = \sum_m c_jk_m rho_m
        constraints = [
            p_jk >= 0,
            np.matrix(np.ones((1, pi_basis.dim))) * p_jk_mat == 1,
        ]

        rho_m_real_imag = sum((rm * o_ut.to_realimag(Pm)
                               for (rm, Pm) in ut.izip(rho_m, pauli_basis.ops)), 0)

        if POSITIVE in settings.constraints:
            if tomography._SDP_SOLVER.is_functional():
                constraints.append(rho_m_real_imag >> 0)
            else:  # pragma no coverage
                _log.warning("No convex solver capable of semi-definite problems installed.\n"
                             "Dropping the positivity constraint on the density matrix.")

        if UNIT_TRACE in settings.constraints:
            # this assumes that the first element of the Pauli basis is always proportional to
            # the identity
            constraints.append(rho_m[0, 0] == 1. / pauli_basis.ops[0].tr().real)

        prob = cvxpy.Problem(cvxpy.Minimize(obj), constraints)

        _log.info("Starting convex solver")
        prob.solve(solver=tomography.SOLVER, **settings.solver_kwargs)
        if prob.status != cvxpy.OPTIMAL:  # pragma no coverage
            _log.warning("Problem did not converge to optimal solution. "
                         "Solver settings: {}".format(settings.solver_kwargs))

        return StateTomography(np.array(rho_m.value).ravel(), pauli_basis, settings)

    def __init__(self, rho_coeffs, pauli_basis, settings):
        """
        Construct a StateTomography to encapsulate the result of estimating the quantum state from
        a quantum tomography measurement.

        :param numpy.ndarray r_est: The estimated quantum state represented in a given (generalized)
        Pauli basis.
        :param OperatorBasis pauli_basis: The employed (generalized) Pauli basis.
        :param TomographySettings settings: The settings used to estimate the state.
        """
        self.rho_coeffs = rho_coeffs
        self.pauli_basis = pauli_basis
        self.rho_est = sum((r_m * p_m for r_m, p_m in ut.izip(rho_coeffs, pauli_basis.ops)))
        self.settings = settings

    def fidelity(self, other):
        """
        Compute the quantum state fidelity of the estimated state with another state.

        :param qutip.Qobj other: The other quantum state.
        :return: The fidelity, a real number between 0 and 1.
        :rtype: float
        """
        return qt.fidelity(self.rho_est, other)

    def plot_state_histogram(self, ax):
        """
        Visualize the complex matrix elements of the estimated state.

        :param matplotlib.Axes ax: A matplotlib Axes object to plot into.
        """
        title = "Estimated state"
        nqc = int(round(np.log2(self.rho_est.data.shape[0])))
        labels = ut.basis_labels(nqc)
        return ut.state_histogram(self.rho_est, ax, title)

    def plot(self):
        """
        Visualize the state.

        :return: The generated figure.
        :rtype: matplotlib.Figure
        """
        width = 10
        # The pleasing golden ratio.
        height = width / 1.618
        f = plt.figure(figsize=(width, height))
        ax = f.add_subplot(111, projection="3d")

        self.plot_state_histogram(ax)
        return f

def state_tomography_programs(state_prep, qubits=None,
                              rotation_generator=tomography.default_rotations):
    """
    Yield tomographic sequences that prepare a state with Quil program `state_prep` and then append
    tomographic rotations on the specified `qubits`. If `qubits is None`, it assumes all qubits in
    the program should be tomographically rotated.

    :param Program state_prep: The program to prepare the state to be tomographed.
    :param list|NoneType qubits: A list of Qubits or Numbers, to perform the tomography on. If
    `None`, performs it on all in state_prep.
    :param generator rotation_generator: A generator that yields tomography rotations to perform.
    :return: Program for state tomography.
    :rtype: Program
    """
    if qubits is None:
        qubits = state_prep.get_qubits()
    for tomography_program in rotation_generator(*qubits):
        state_tomography_program = Program(Pragma("PRESERVE_BLOCK"))
        state_tomography_program.inst(state_prep)
        state_tomography_program.inst(tomography_program)
        state_tomography_program.inst(Pragma("END_PRESERVE_BLOCK"))
        yield state_tomography_program

def do_state_tomography(preparation_program, nsamples, cxn, qubits=None, use_run=False):
    """
    Method to perform both a QPU and QVM state tomography, and use the latter as
    as reference to calculate the fidelity of the former.

    :param Program preparation_program: Program to execute.
    :param int nsamples: Number of samples to take for the program.
    :param QVMConnection|QPUConnection cxn: Connection on which to run the program.
    :param list qubits: List of qubits for the program.
    to use in the tomography analysis.
    :param bool use_run: If ``True``, use append measurements on all qubits and use ``cxn.run``
        instead of ``cxn.run_and_measure``.
    :return: The state tomogram.
    :rtype: StateTomography
    """
    return tomography._do_tomography(preparation_program, nsamples, cxn, qubits,
                                     tomography.MAX_QUBITS_STATE_TOMO,
                                     StateTomography, state_tomography_programs,
                                     DEFAULT_STATE_TOMO_SETTINGS, use_run=use_run)


import gzip
import json
import os
import shutil
import hashlib
from os.path import join
from warnings import warn
from contextlib import closing
from functools import wraps
from typing import Callable, Optional, Dict, Tuple, List, Any, Union
import itertools
from collections.abc import Generator
from collections import OrderedDict
from functools import partial

from urllib.request import urlopen, Request

import numpy as np
import scipy.sparse

from ..externals import _arff
from ..externals._arff import ArffSparseDataType, ArffContainerType
from . import get_data_home
from urllib.error import HTTPError
from ..utils import Bunch
from ..utils import is_scalar_nan
from ..utils import get_chunk_n_rows
from ..utils import _chunk_generator
from ..utils import check_pandas_support  # noqa

__all__ = ['fetch_openml']

_OPENML_PREFIX = "https://openml.org/"
_SEARCH_NAME = "api/v1/json/data/list/data_name/{}/limit/2"
_DATA_INFO = "api/v1/json/data/{}"
_DATA_FEATURES = "api/v1/json/data/features/{}"
_DATA_QUALITIES = "api/v1/json/data/qualities/{}"
_DATA_FILE = "data/v1/download/{}"

OpenmlQualitiesType = List[Dict[str, str]]
OpenmlFeaturesType = List[Dict[str, str]]

def _get_local_path(openml_path: str, data_home: str) -> str:
    return os.path.join(data_home, 'openml.org', openml_path + ".gz")

def _retry_with_clean_cache(
    openml_path: str, data_home: Optional[str]
) -> Callable:
    """If the first call to the decorated function fails, the local cached
    file is removed, and the function is called again. If ``data_home`` is
    ``None``, then the function is called once.
    """
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kw):
            if data_home is None:
                return f(*args, **kw)
            try:
                return f(*args, **kw)
            except HTTPError:
                raise
            except Exception:
                warn("Invalid cache, redownloading file", RuntimeWarning)
                local_path = _get_local_path(openml_path, data_home)
                if os.path.exists(local_path):
                    os.unlink(local_path)
                return f(*args, **kw)
        return wrapper
    return decorator

def _open_openml_url(openml_path: str, data_home: Optional[str]):
    """
    Returns a resource from OpenML.org. Caches it to data_home if required.

    Parameters
    ----------
    openml_path : str
        OpenML URL that will be accessed. This will be prefixes with
        _OPENML_PREFIX

    data_home : str
        Directory to which the files will be cached. If None, no caching will
        be applied.

    Returns
    -------
    result : stream
        A stream to the OpenML resource
    """
    def is_gzip_encoded(_fsrc):
        return _fsrc.info().get('Content-Encoding', '') == 'gzip'

    req = Request(_OPENML_PREFIX + openml_path)
    req.add_header('Accept-encoding', 'gzip')

    if data_home is None:
        fsrc = urlopen(req)
        if is_gzip_encoded(fsrc):
            return gzip.GzipFile(fileobj=fsrc, mode='rb')
        return fsrc

    local_path = _get_local_path(openml_path, data_home)
    if not os.path.exists(local_path):
        try:
            os.makedirs(os.path.dirname(local_path))
        except OSError:
            # potentially, the directory has been created already
            pass

        try:
            with closing(urlopen(req)) as fsrc:
                opener: Callable
                if is_gzip_encoded(fsrc):
                    opener = open
                else:
                    opener = gzip.GzipFile
                with opener(local_path, 'wb') as fdst:
                    shutil.copyfileobj(fsrc, fdst)
        except Exception:
            if os.path.exists(local_path):
                os.unlink(local_path)
            raise

    # XXX: First time, decompression will not be necessary (by using fsrc), but
    # it will happen nonetheless
    return gzip.GzipFile(local_path, 'rb')

class OpenMLError(ValueError):
    """HTTP 412 is a specific OpenML error code, indicating a generic error"""
    pass

def _get_json_content_from_openml_api(
    url: str,
    error_message: Optional[str],
    data_home: Optional[str]
) -> Dict:
    """
    Loads json data from the openml api

    Parameters
    ----------
    url : str
        The URL to load from. Should be an official OpenML endpoint

    error_message : str or None
        The error message to raise if an acceptable OpenML error is thrown
        (acceptable error is, e.g., data id not found. Other errors, like 404's
        will throw the native error message)

    data_home : str or None
        Location to cache the response. None if no cache is required.

    Returns
    -------
    json_data : json
        the json result from the OpenML server if the call was successful.
        An exception otherwise.
    """

    @_retry_with_clean_cache(url, data_home)
    def _load_json():
        with closing(_open_openml_url(url, data_home)) as response:
            return json.loads(response.read().decode("utf-8"))

    try:
        return _load_json()
    except HTTPError as error:
        # 412 is an OpenML specific error code, indicating a generic error
        # (e.g., data not found)
        if error.code != 412:
            raise error

    # 412 error, not in except for nicer traceback
    raise OpenMLError(error_message)

def _split_sparse_columns(
    arff_data: ArffSparseDataType, include_columns: List
) -> ArffSparseDataType:
    """
    obtains several columns from sparse arff representation. Additionally, the
    column indices are re-labelled, given the columns that are not included.
    (e.g., when including [1, 2, 3], the columns will be relabelled to
    [0, 1, 2])

    Parameters
    ----------
    arff_data : tuple
        A tuple of three lists of equal size; first list indicating the value,
        second the x coordinate and the third the y coordinate.

    include_columns : list
        A list of columns to include.

    Returns
    -------
    arff_data_new : tuple
        Subset of arff data with only the include columns indicated by the
        include_columns argument.
    """
    arff_data_new: ArffSparseDataType = (list(), list(), list())
    reindexed_columns = {column_idx: array_idx for array_idx, column_idx
                         in enumerate(include_columns)}
    for val, row_idx, col_idx in zip(arff_data[0], arff_data[1], arff_data[2]):
        if col_idx in include_columns:
            arff_data_new[0].append(val)
            arff_data_new[1].append(row_idx)
            arff_data_new[2].append(reindexed_columns[col_idx])
    return arff_data_new

def _sparse_data_to_array(
    arff_data: ArffSparseDataType, include_columns: List
) -> np.ndarray:
    # turns the sparse data back into an array (can't use toarray() function,
    # as this does only work on numeric data)
    num_obs = max(arff_data[1]) + 1
    y_shape = (num_obs, len(include_columns))
    reindexed_columns = {column_idx: array_idx for array_idx, column_idx
                         in enumerate(include_columns)}
    # TODO: improve for efficiency
    y = np.empty(y_shape, dtype=np.float64)
    for val, row_idx, col_idx in zip(arff_data[0], arff_data[1], arff_data[2]):
        if col_idx in include_columns:
            y[row_idx, reindexed_columns[col_idx]] = val
    return y

def _convert_arff_data(
    arff: ArffContainerType,
    col_slice_x: List[int],
    col_slice_y: List[int],
    shape: Optional[Tuple] = None
) -> Tuple:
    """
    converts the arff object into the appropriate matrix type (np.array or
    scipy.sparse.csr_matrix) based on the 'data part' (i.e., in the
    liac-arff dict, the object from the 'data' key)

    Parameters
    ----------
    arff : dict
        As obtained from liac-arff object.

    col_slice_x : list
        The column indices that are sliced from the original array to return
        as X data

    col_slice_y : list
        The column indices that are sliced from the original array to return
        as y data

    Returns
    -------
    X : np.array or scipy.sparse.csr_matrix
    y : np.array
    """
    arff_data = arff['data']
    if isinstance(arff_data, Generator):
        if shape is None:
            raise ValueError(
                "shape must be provided when arr['data'] is a Generator"
            )
        if shape[0] == -1:
            count = -1
        else:
            count = shape[0] * shape[1]
        data = np.fromiter(itertools.chain.from_iterable(arff_data),
                           dtype='float64', count=count)
        data = data.reshape(*shape)
        X = data[:, col_slice_x]
        y = data[:, col_slice_y]
        return X, y
    elif isinstance(arff_data, tuple):
        arff_data_X = _split_sparse_columns(arff_data, col_slice_x)
        num_obs = max(arff_data[1]) + 1
        X_shape = (num_obs, len(col_slice_x))
        X = scipy.sparse.coo_matrix(
            (arff_data_X[0], (arff_data_X[1], arff_data_X[2])),
            shape=X_shape, dtype=np.float64)
        X = X.tocsr()
        y = _sparse_data_to_array(arff_data, col_slice_y)
        return X, y
    else:
        # This should never happen
        raise ValueError('Unexpected Data Type obtained from arff.')

def _feature_to_dtype(feature: Dict[str, str]):
    """Map feature to dtype for pandas DataFrame
    """
    if feature['data_type'] == 'string':
        return object
    elif feature['data_type'] == 'nominal':
        return 'category'
    # only numeric, integer, real are left
    elif (feature['number_of_missing_values'] != '0' or
          feature['data_type'] in ['numeric', 'real']):
        # cast to floats when there are any missing values
        return np.float64
    elif feature['data_type'] == 'integer':
        return np.int64
    raise ValueError('Unsupported feature: {}'.format(feature))

def _convert_arff_data_dataframe(
    arff: ArffContainerType, columns: List, features_dict: Dict[str, Any]
) -> Tuple:
    """Convert the ARFF object into a pandas DataFrame.

    Parameters
    ----------
    arff : dict
        As obtained from liac-arff object.

    columns : list
        Columns from dataframe to return.

    features_dict : dict
        Maps feature name to feature info from openml.

    Returns
    -------
    result : tuple
        tuple with the resulting dataframe
    """
    pd = check_pandas_support('fetch_openml with as_frame=True')

    attributes = OrderedDict(arff['attributes'])
    arff_columns = list(attributes)

    if not isinstance(arff['data'], Generator):
        raise ValueError(
            "arff['data'] must be a generator when converting to pd.DataFrame."
        )

    # calculate chunksize
    first_row = next(arff['data'])
    first_df = pd.DataFrame([first_row], columns=arff_columns)

    row_bytes = first_df.memory_usage(deep=True).sum()
    chunksize = get_chunk_n_rows(row_bytes)

    # read arff data with chunks
    columns_to_keep = [col for col in arff_columns if col in columns]
    dfs = []
    dfs.append(first_df[columns_to_keep])
    for data in _chunk_generator(arff['data'], chunksize):
        dfs.append(pd.DataFrame(data, columns=arff_columns)[columns_to_keep])
    df = pd.concat(dfs, ignore_index=True)

    for column in columns_to_keep:
        dtype = _feature_to_dtype(features_dict[column])
        if dtype == 'category':
            cats_without_missing = [cat for cat in attributes[column]
                                    if cat is not None and
                                    not is_scalar_nan(cat)]
            dtype = pd.api.types.CategoricalDtype(cats_without_missing)
        df[column] = df[column].astype(dtype, copy=False)
    return (df, )

def _get_data_info_by_name(
    name: str, version: Union[int, str], data_home: Optional[str]
):
    """
    Utilizes the openml dataset listing api to find a dataset by
    name/version
    OpenML api function:
    https://www.openml.org/api_docs#!/data/get_data_list_data_name_data_name

    Parameters
    ----------
    name : str
        name of the dataset

    version : int or str
        If version is an integer, the exact name/version will be obtained from
        OpenML. If version is a string (value: "active") it will take the first
        version from OpenML that is annotated as active. Any other string
        values except "active" are treated as integer.

    data_home : str or None
        Location to cache the response. None if no cache is required.

    Returns
    -------
    first_dataset : json
        json representation of the first dataset object that adhired to the
        search criteria

    """
    if version == "active":
        # situation in which we return the oldest active version
        url = _SEARCH_NAME.format(name) + "/status/active/"
        error_msg = "No active dataset {} found.".format(name)
        json_data = _get_json_content_from_openml_api(
            url, error_msg, data_home=data_home
        )
        res = json_data['data']['dataset']
        if len(res) > 1:
            warn("Multiple active versions of the dataset matching the name"
                 " {name} exist. Versions may be fundamentally different, "
                 "returning version"
                 " {version}.".format(name=name, version=res[0]['version']))
        return res[0]

    # an integer version has been provided
    url = (_SEARCH_NAME + "/data_version/{}").format(name, version)
    try:
        json_data = _get_json_content_from_openml_api(
            url, error_message=None, data_home=data_home
        )
    except OpenMLError:
        # we can do this in 1 function call if OpenML does not require the
        # specification of the dataset status (i.e., return datasets with a
        # given name / version regardless of active, deactivated, etc. )
        # TODO: feature request OpenML.
        url += "/status/deactivated"
        error_msg = "Dataset {} with version {} not found.".format(name,
                                                                   version)
        json_data = _get_json_content_from_openml_api(
            url, error_msg, data_home=data_home
        )

    return json_data['data']['dataset'][0]

def _get_data_description_by_id(
    data_id: int, data_home: Optional[str]
) -> Dict[str, Any]:
    # OpenML API function: https://www.openml.org/api_docs#!/data/get_data_id
    url = _DATA_INFO.format(data_id)
    error_message = "Dataset with data_id {} not found.".format(data_id)
    json_data = _get_json_content_from_openml_api(
        url, error_message, data_home=data_home
    )
    return json_data['data_set_description']

def _get_data_features(
    data_id: int, data_home: Optional[str]
) -> OpenmlFeaturesType:
    # OpenML function:
    # https://www.openml.org/api_docs#!/data/get_data_features_id
    url = _DATA_FEATURES.format(data_id)
    error_message = "Dataset with data_id {} not found.".format(data_id)
    json_data = _get_json_content_from_openml_api(
        url, error_message, data_home=data_home
    )
    return json_data['data_features']['feature']

def _get_data_qualities(
    data_id: int, data_home: Optional[str]
) -> OpenmlQualitiesType:
    # OpenML API function:
    # https://www.openml.org/api_docs#!/data/get_data_qualities_id
    url = _DATA_QUALITIES.format(data_id)
    error_message = "Dataset with data_id {} not found.".format(data_id)
    json_data = _get_json_content_from_openml_api(
        url, error_message, data_home=data_home
    )
    # the qualities might not be available, but we still try to process
    # the data
    return json_data.get('data_qualities', {}).get('quality', [])

def _get_num_samples(data_qualities: OpenmlQualitiesType) -> int:
    """Get the number of samples from data qualities.

    Parameters
    ----------
    data_qualities : list of dict
        Used to retrieve the number of instances (samples) in the dataset.

    Returns
    -------
    n_samples : int
        The number of samples in the dataset or -1 if data qualities are
        unavailable.
    """
    # If the data qualities are unavailable, we return -1
    default_n_samples = -1

    qualities = {d['name']: d['value'] for d in data_qualities}
    return int(float(qualities.get('NumberOfInstances', default_n_samples)))

def _load_arff_response(
    url: str,
    data_home: Optional[str],
    return_type, encode_nominal: bool,
    parse_arff: Callable[[ArffContainerType], Tuple],
    md5_checksum: str
) -> Tuple:
    """Load arff data with url and parses arff response with parse_arff"""
    response = _open_openml_url(url, data_home)

    with closing(response):
        # Note that if the data is dense, no reading is done until the data
        # generator is iterated.
        actual_md5_checksum = hashlib.md5()

        def _stream_checksum_generator(response):
            for line in response:
                actual_md5_checksum.update(line)
                yield line.decode('utf-8')

        stream = _stream_checksum_generator(response)

        arff = _arff.load(stream,
                          return_type=return_type,
                          encode_nominal=encode_nominal)

        parsed_arff = parse_arff(arff)

        # consume remaining stream, if early exited
        for _ in stream:
            pass

        if actual_md5_checksum.hexdigest() != md5_checksum:
            raise ValueError("md5 checksum of local file for " + url +
                             " does not match description. "
                             "Downloaded file could have been modified / "
                             "corrupted, clean cache and retry...")

        return parsed_arff

def _download_data_to_bunch(
    url: str,
    sparse: bool,
    data_home: Optional[str],
    *,
    as_frame: bool,
    features_list: List,
    data_columns: List[int],
    target_columns: List,
    shape: Optional[Tuple[int, int]],
    md5_checksum: str
):
    """Download OpenML ARFF and convert to Bunch of data
    """
    # NB: this function is long in order to handle retry for any failure
    #     during the streaming parse of the ARFF.

    # Prepare which columns and data types should be returned for the X and y
    features_dict = {feature['name']: feature for feature in features_list}

    # XXX: col_slice_y should be all nominal or all numeric
    _verify_target_data_type(features_dict, target_columns)

    col_slice_y = [int(features_dict[col_name]['index'])
                   for col_name in target_columns]

    col_slice_x = [int(features_dict[col_name]['index'])
                   for col_name in data_columns]
    for col_idx in col_slice_y:
        feat = features_list[col_idx]
        nr_missing = int(feat['number_of_missing_values'])
        if nr_missing > 0:
            raise ValueError('Target column {} has {} missing values. '
                             'Missing values are not supported for target '
                             'columns. '.format(feat['name'], nr_missing))

    # Access an ARFF file on the OpenML server. Documentation:
    # https://www.openml.org/api_data_docs#!/data/get_download_id

    if sparse is True:
        return_type = _arff.COO
    else:
        return_type = _arff.DENSE_GEN

    frame = nominal_attributes = None

    parse_arff: Callable
    postprocess: Callable
    if as_frame:
        columns = data_columns + target_columns
        parse_arff = partial(_convert_arff_data_dataframe, columns=columns,
                             features_dict=features_dict)

        def postprocess(frame):
            X = frame[data_columns]
            if len(target_columns) >= 2:
                y = frame[target_columns]
            elif len(target_columns) == 1:
                y = frame[target_columns[0]]
            else:
                y = None
            return X, y, frame, nominal_attributes
    else:
        def parse_arff(arff):
            X, y = _convert_arff_data(arff, col_slice_x, col_slice_y, shape)
            # nominal attributes is a dict mapping from the attribute name to
            # the possible values. Includes also the target column (which will
            # be popped off below, before it will be packed in the Bunch
            # object)
            nominal_attributes = {k: v for k, v in arff['attributes']
                                  if isinstance(v, list) and
                                  k in data_columns + target_columns}
            return X, y, nominal_attributes

        def postprocess(X, y, nominal_attributes):
            is_classification = {col_name in nominal_attributes
                                 for col_name in target_columns}
            if not is_classification:
                # No target
                pass
            elif all(is_classification):
                y = np.hstack([
                    np.take(
                        np.asarray(nominal_attributes.pop(col_name),
                                   dtype='O'),
                        y[:, i:i + 1].astype(int, copy=False))
                    for i, col_name in enumerate(target_columns)
                ])
            elif any(is_classification):
                raise ValueError('Mix of nominal and non-nominal targets is '
                                 'not currently supported')

            # reshape y back to 1-D array, if there is only 1 target column;
            # back to None if there are not target columns
            if y.shape[1] == 1:
                y = y.reshape((-1,))
            elif y.shape[1] == 0:
                y = None
            return X, y, frame, nominal_attributes

    out = _retry_with_clean_cache(url, data_home)(
        _load_arff_response)(url, data_home,
                             return_type=return_type,
                             encode_nominal=not as_frame,
                             parse_arff=parse_arff,
                             md5_checksum=md5_checksum)
    X, y, frame, nominal_attributes = postprocess(*out)

    return Bunch(data=X, target=y, frame=frame,
                 categories=nominal_attributes,
                 feature_names=data_columns,
                 target_names=target_columns)

def _verify_target_data_type(features_dict, target_columns):
    # verifies the data type of the y array in case there are multiple targets
    # (throws an error if these targets do not comply with sklearn support)
    if not isinstance(target_columns, list):
        raise ValueError('target_column should be list, '
                         'got: %s' % type(target_columns))
    found_types = set()
    for target_column in target_columns:
        if target_column not in features_dict:
            raise KeyError('Could not find target_column={}')
        if features_dict[target_column]['data_type'] == "numeric":
            found_types.add(np.float64)
        else:
            found_types.add(object)

        # note: we compare to a string, not boolean
        if features_dict[target_column]['is_ignore'] == 'true':
            warn('target_column={} has flag is_ignore.'.format(
                target_column))
        if features_dict[target_column]['is_row_identifier'] == 'true':
            warn('target_column={} has flag is_row_identifier.'.format(
                target_column))
    if len(found_types) > 1:
        raise ValueError('Can only handle homogeneous multi-target datasets, '
                         'i.e., all targets are either numeric or '
                         'categorical.')

def _valid_data_column_names(features_list, target_columns):
    # logic for determining on which columns can be learned. Note that from the
    # OpenML guide follows that columns that have the `is_row_identifier` or
    # `is_ignore` flag, these can not be learned on. Also target columns are
    # excluded.
    valid_data_column_names = []
    for feature in features_list:
        if (feature['name'] not in target_columns
                and feature['is_ignore'] != 'true'
                and feature['is_row_identifier'] != 'true'):
            valid_data_column_names.append(feature['name'])
    return valid_data_column_names

def fetch_openml(
    name: Optional[str] = None,
    *,
    version: Union[str, int] = 'active',
    data_id: Optional[int] = None,
    data_home: Optional[str] = None,
    target_column: Optional[Union[str, List]] = 'default-target',
    cache: bool = True,
    return_X_y: bool = False,
    as_frame: Union[str, bool] = 'auto'
):
    """Fetch dataset from openml by name or dataset id.

    Datasets are uniquely identified by either an integer ID or by a
    combination of name and version (i.e. there might be multiple
    versions of the 'iris' dataset). Please give either name or data_id
    (not both). In case a name is given, a version can also be
    provided.

    Read more in the :ref:`User Guide <openml>`.

    .. versionadded:: 0.20

    .. note:: EXPERIMENTAL

        The API is experimental (particularly the return value structure),
        and might have small backward-incompatible changes without notice
        or warning in future releases.

    Parameters
    ----------
    name : str, default=None
        String identifier of the dataset. Note that OpenML can have multiple
        datasets with the same name.

    version : int or 'active', default='active'
        Version of the dataset. Can only be provided if also ``name`` is given.
        If 'active' the oldest version that's still active is used. Since
        there may be more than one active version of a dataset, and those
        versions may fundamentally be different from one another, setting an
        exact version is highly recommended.

    data_id : int, default=None
        OpenML ID of the dataset. The most specific way of retrieving a
        dataset. If data_id is not given, name (and potential version) are
        used to obtain a dataset.

    data_home : str, default=None
        Specify another download and cache folder for the data sets. By default
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    target_column : str, list or None, default='default-target'
        Specify the column name in the data to use as target. If
        'default-target', the standard target column a stored on the server
        is used. If ``None``, all columns are returned as data and the
        target is ``None``. If list (of strings), all columns with these names
        are returned as multi-target (Note: not all scikit-learn classifiers
        can handle all types of multi-output combinations)

    cache : bool, default=True
        Whether to cache downloaded datasets using joblib.

    return_X_y : bool, default=False
        If True, returns ``(data, target)`` instead of a Bunch object. See
        below for more information about the `data` and `target` objects.

    as_frame : bool or 'auto', default='auto'
        If True, the data is a pandas DataFrame including columns with
        appropriate dtypes (numeric, string or categorical). The target is
        a pandas DataFrame or Series depending on the number of target_columns.
        The Bunch will contain a ``frame`` attribute with the target and the
        data. If ``return_X_y`` is True, then ``(data, target)`` will be pandas
        DataFrames or Series as describe above.

        If as_frame is 'auto', the data and target will be converted to
        DataFrame or Series as if as_frame is set to True, unless the dataset
        is stored in sparse format.

        .. versionchanged:: 0.24
           The default value of `as_frame` changed from `False` to `'auto'`
           in 0.24.

    Returns
    -------

    data : :class:`~sklearn.utils.Bunch`
        Dictionary-like object, with the following attributes.

        data : np.array, scipy.sparse.csr_matrix of floats, or pandas DataFrame
            The feature matrix. Categorical features are encoded as ordinals.
        target : np.array, pandas Series or DataFrame
            The regression target or classification labels, if applicable.
            Dtype is float if numeric, and object if categorical. If
            ``as_frame`` is True, ``target`` is a pandas object.
        DESCR : str
            The full description of the dataset
        feature_names : list
            The names of the dataset columns
        target_names: list
            The names of the target columns

        .. versionadded:: 0.22

        categories : dict or None
            Maps each categorical feature name to a list of values, such
            that the value encoded as i is ith in the list. If ``as_frame``
            is True, this is None.
        details : dict
            More metadata from OpenML
        frame : pandas DataFrame
            Only present when `as_frame=True`. DataFrame with ``data`` and
            ``target``.

    (data, target) : tuple if ``return_X_y`` is True

        .. note:: EXPERIMENTAL

            This interface is **experimental** and subsequent releases may
            change attributes without notice (although there should only be
            minor changes to ``data`` and ``target``).

        Missing values in the 'data' are represented as NaN's. Missing values
        in 'target' are represented as NaN's (numerical target) or None
        (categorical target)
    """
    if cache is False:
        # no caching will be applied
        data_home = None
    else:
        data_home = get_data_home(data_home=data_home)
        data_home = join(data_home, 'openml')

    # check valid function arguments. data_id XOR (name, version) should be
    # provided
    if name is not None:
        # OpenML is case-insensitive, but the caching mechanism is not
        # convert all data names (str) to lower case
        name = name.lower()
        if data_id is not None:
            raise ValueError(
                "Dataset data_id={} and name={} passed, but you can only "
                "specify a numeric data_id or a name, not "
                "both.".format(data_id, name))
        data_info = _get_data_info_by_name(name, version, data_home)
        data_id = data_info['did']
    elif data_id is not None:
        # from the previous if statement, it is given that name is None
        if version != "active":
            raise ValueError(
                "Dataset data_id={} and version={} passed, but you can only "
                "specify a numeric data_id or a version, not "
                "both.".format(data_id, version))
    else:
        raise ValueError(
            "Neither name nor data_id are provided. Please provide name or "
            "data_id.")

    data_description = _get_data_description_by_id(data_id, data_home)
    if data_description['status'] != "active":
        warn("Version {} of dataset {} is inactive, meaning that issues have "
             "been found in the dataset. Try using a newer version from "
             "this URL: {}".format(
                data_description['version'],
                data_description['name'],
                data_description['url']))
    if 'error' in data_description:
        warn("OpenML registered a problem with the dataset. It might be "
             "unusable. Error: {}".format(data_description['error']))
    if 'warning' in data_description:
        warn("OpenML raised a warning on the dataset. It might be "
             "unusable. Warning: {}".format(data_description['warning']))

    return_sparse = False
    if data_description['format'].lower() == 'sparse_arff':
        return_sparse = True

    if as_frame == 'auto':
        as_frame = not return_sparse

    if as_frame and return_sparse:
        raise ValueError('Cannot return dataframe with sparse data')

    # download data features, meta-info about column types
    features_list = _get_data_features(data_id, data_home)

    if not as_frame:
        for feature in features_list:
            if 'true' in (feature['is_ignore'], feature['is_row_identifier']):
                continue
            if feature['data_type'] == 'string':
                raise ValueError('STRING attributes are not supported for '
                                 'array representation. Try as_frame=True')

    if target_column == "default-target":
        # determines the default target based on the data feature results
        # (which is currently more reliable than the data description;
        # see issue: https://github.com/openml/OpenML/issues/768)
        target_columns = [feature['name'] for feature in features_list
                          if feature['is_target'] == 'true']
    elif isinstance(target_column, str):
        # for code-simplicity, make target_column by default a list
        target_columns = [target_column]
    elif target_column is None:
        target_columns = []
    elif isinstance(target_column, list):
        target_columns = target_column
    else:
        raise TypeError("Did not recognize type of target_column"
                        "Should be str, list or None. Got: "
                        "{}".format(type(target_column)))
    data_columns = _valid_data_column_names(features_list,
                                            target_columns)

    shape: Optional[Tuple[int, int]]
    # determine arff encoding to return
    if not return_sparse:
        # The shape must include the ignored features to keep the right indexes
        # during the arff data conversion.
        data_qualities = _get_data_qualities(data_id, data_home)
        shape = _get_num_samples(data_qualities), len(features_list)
    else:
        shape = None

    # obtain the data
    url = _DATA_FILE.format(data_description['file_id'])
    bunch = _download_data_to_bunch(url, return_sparse, data_home,
                                    as_frame=bool(as_frame),
                                    features_list=features_list, shape=shape,
                                    target_columns=target_columns,
                                    data_columns=data_columns,
                                    md5_checksum=data_description[
                                        "md5_checksum"])

    if return_X_y:
        return bunch.data, bunch.target

    description = "{}\n\nDownloaded from openml.org.".format(
        data_description.pop('description'))

    bunch.update(
        DESCR=description, details=data_description,
        url="https://www.openml.org/d/{}".format(data_id))

    return bunch