# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause
import os
from os.path import join

import numpy

from sklearn._build_utils import get_blas_info

def configuration(parent_package='', top_path=None):
    from numpy.distutils.misc_util import Configuration

    cblas_libs, blas_info = get_blas_info()

    libraries = []
    if os.name == 'posix':
        cblas_libs.append('m')
        libraries.append('m')

    config = Configuration('cluster', parent_package, top_path)
    config.add_extension('_dbscan_inner',
                         sources=['_dbscan_inner.cpp'],
                         include_dirs=[numpy.get_include()],
                         language="c++")

    config.add_extension('_hierarchical',
                         sources=['_hierarchical.cpp'],
                         language="c++",
                         include_dirs=[numpy.get_include()],
                         libraries=libraries)

    config.add_extension(
        '_k_means',
        libraries=cblas_libs,
        sources=['_k_means.c'],
        include_dirs=[join('..', 'src', 'cblas'),
                      numpy.get_include(),
                      blas_info.pop('include_dirs', [])],
        extra_compile_args=blas_info.pop('extra_compile_args', []),
        **blas_info
    )

    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())


"""
===================================================================
Multi-output Decision Tree Regression
===================================================================

An example to illustrate multi-output regression with decision tree.

The :ref:`decision trees <tree>`
is used to predict simultaneously the noisy x and y observations of a circle
given a single underlying feature. As a result, it learns local linear
regressions approximating the circle.

We can see that if the maximum depth of the tree (controlled by the
`max_depth` parameter) is set too high, the decision trees learn too fine
details of the training data and learn from the noise, i.e. they overfit.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor

# Create a random dataset
rng = np.random.RandomState(1)
X = np.sort(200 * rng.rand(100, 1) - 100, axis=0)
y = np.array([np.pi * np.sin(X).ravel(), np.pi * np.cos(X).ravel()]).T
y[::5, :] += (0.5 - rng.rand(20, 2))

# Fit regression model
regr_1 = DecisionTreeRegressor(max_depth=2)
regr_2 = DecisionTreeRegressor(max_depth=5)
regr_3 = DecisionTreeRegressor(max_depth=8)
regr_1.fit(X, y)
regr_2.fit(X, y)
regr_3.fit(X, y)

# Predict
X_test = np.arange(-100.0, 100.0, 0.01)[:, np.newaxis]
y_1 = regr_1.predict(X_test)
y_2 = regr_2.predict(X_test)
y_3 = regr_3.predict(X_test)

# Plot the results
plt.figure()
plt.scatter(y[:, 0], y[:, 1], c="k", label="data")
plt.scatter(y_1[:, 0], y_1[:, 1], c="g", label="max_depth=2")
plt.scatter(y_2[:, 0], y_2[:, 1], c="r", label="max_depth=5")
plt.scatter(y_3[:, 0], y_3[:, 1], c="b", label="max_depth=8")
plt.xlim([-6, 6])
plt.ylim([-6, 6])
plt.xlabel("data")
plt.ylabel("target")
plt.title("Multi-output Decision Tree Regression")
plt.legend()
plt.show()


#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""Example of recurrent neural networks over characters for DBpedia dataset.

This model is similar to one described in this paper:
   "Character-level Convolutional Networks for Text Classification"
   http://arxiv.org/abs/1509.01626

and is somewhat alternative to the Lua code from here:
   https://github.com/zhangxiangxiao/Crepe
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import sys

import numpy as np
import pandas
import tensorflow as tf

FLAGS = None

MAX_DOCUMENT_LENGTH = 100
HIDDEN_SIZE = 20
MAX_LABEL = 15
CHARS_FEATURE = 'chars'  # Name of the input character feature.

def char_rnn_model(features, labels, mode):
  """Character level recurrent neural network model to predict classes."""
  byte_vectors = tf.one_hot(features[CHARS_FEATURE], 256, 1., 0.)
  byte_list = tf.unstack(byte_vectors, axis=1)

  cell = tf.nn.rnn_cell.GRUCell(HIDDEN_SIZE)
  _, encoding = tf.nn.static_rnn(cell, byte_list, dtype=tf.float32)

  logits = tf.layers.dense(encoding, MAX_LABEL, activation=None)

  predicted_classes = tf.argmax(logits, 1)
  if mode == tf.estimator.ModeKeys.PREDICT:
    return tf.estimator.EstimatorSpec(
        mode=mode,
        predictions={
            'class': predicted_classes,
            'prob': tf.nn.softmax(logits)
        })

  onehot_labels = tf.one_hot(labels, MAX_LABEL, 1, 0)
  loss = tf.losses.softmax_cross_entropy(
      onehot_labels=onehot_labels, logits=logits)
  if mode == tf.estimator.ModeKeys.TRAIN:
    optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
    train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
    return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)

  eval_metric_ops = {
      'accuracy': tf.metrics.accuracy(
          labels=labels, predictions=predicted_classes)
  }
  return tf.estimator.EstimatorSpec(
      mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)

def main(unused_argv):
  # Prepare training and testing data
  dbpedia = tf.contrib.learn.datasets.load_dataset(
      'dbpedia', test_with_fake_data=FLAGS.test_with_fake_data)
  x_train = pandas.DataFrame(dbpedia.train.data)[1]
  y_train = pandas.Series(dbpedia.train.target)
  x_test = pandas.DataFrame(dbpedia.test.data)[1]
  y_test = pandas.Series(dbpedia.test.target)

  # Process vocabulary
  char_processor = tf.contrib.learn.preprocessing.ByteProcessor(
      MAX_DOCUMENT_LENGTH)
  x_train = np.array(list(char_processor.fit_transform(x_train)))
  x_test = np.array(list(char_processor.transform(x_test)))

  # Build model
  classifier = tf.estimator.Estimator(model_fn=char_rnn_model)

  # Train.
  train_input_fn = tf.estimator.inputs.numpy_input_fn(
      x={CHARS_FEATURE: x_train},
      y=y_train,
      batch_size=128,
      num_epochs=None,
      shuffle=True)
  classifier.train(input_fn=train_input_fn, steps=100)

  # Eval.
  test_input_fn = tf.estimator.inputs.numpy_input_fn(
      x={CHARS_FEATURE: x_test},
      y=y_test,
      num_epochs=1,
      shuffle=False)

  scores = classifier.evaluate(input_fn=test_input_fn)
  print('Accuracy: {0:f}'.format(scores['accuracy']))

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--test_with_fake_data',
      default=False,
      help='Test the example code with fake data.',
      action='store_true')
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


# -*- coding: utf-8 -*-
import nose

from pandas.core import panelnd
from pandas.core.panel import Panel

from pandas.util.testing import assert_panel_equal
import pandas.util.testing as tm

class TestPanelnd(tm.TestCase):

    def setUp(self):
        pass

    def test_4d_construction(self):

        # create a 4D
        Panel4D = panelnd.create_nd_panel_factory(
            klass_name='Panel4D',
            orders=['labels', 'items', 'major_axis', 'minor_axis'],
            slices={'items': 'items', 'major_axis': 'major_axis',
                    'minor_axis': 'minor_axis'},
            slicer=Panel,
            aliases={'major': 'major_axis', 'minor': 'minor_axis'},
            stat_axis=2)

        p4d = Panel4D(dict(L1=tm.makePanel(), L2=tm.makePanel()))  # noqa

    def test_4d_construction_alt(self):

        # create a 4D
        Panel4D = panelnd.create_nd_panel_factory(
            klass_name='Panel4D',
            orders=['labels', 'items', 'major_axis', 'minor_axis'],
            slices={'items': 'items', 'major_axis': 'major_axis',
                    'minor_axis': 'minor_axis'},
            slicer='Panel',
            aliases={'major': 'major_axis', 'minor': 'minor_axis'},
            stat_axis=2)

        p4d = Panel4D(dict(L1=tm.makePanel(), L2=tm.makePanel()))  # noqa

    def test_4d_construction_error(self):

        # create a 4D
        self.assertRaises(Exception,
                          panelnd.create_nd_panel_factory,
                          klass_name='Panel4D',
                          orders=['labels', 'items', 'major_axis',
                                  'minor_axis'],
                          slices={'items': 'items',
                                  'major_axis': 'major_axis',
                                  'minor_axis': 'minor_axis'},
                          slicer='foo',
                          aliases={'major': 'major_axis',
                                   'minor': 'minor_axis'},
                          stat_axis=2)

    def test_5d_construction(self):

        # create a 4D
        Panel4D = panelnd.create_nd_panel_factory(
            klass_name='Panel4D',
            orders=['labels1', 'items', 'major_axis', 'minor_axis'],
            slices={'items': 'items', 'major_axis': 'major_axis',
                    'minor_axis': 'minor_axis'},
            slicer=Panel,
            aliases={'major': 'major_axis', 'minor': 'minor_axis'},
            stat_axis=2)

        p4d = Panel4D(dict(L1=tm.makePanel(), L2=tm.makePanel()))

        # create a 5D
        Panel5D = panelnd.create_nd_panel_factory(
            klass_name='Panel5D',
            orders=['cool1', 'labels1', 'items', 'major_axis',
                    'minor_axis'],
            slices={'labels1': 'labels1', 'items': 'items',
                    'major_axis': 'major_axis',
                    'minor_axis': 'minor_axis'},
            slicer=Panel4D,
            aliases={'major': 'major_axis', 'minor': 'minor_axis'},
            stat_axis=2)

        p5d = Panel5D(dict(C1=p4d))

        # slice back to 4d
        results = p5d.ix['C1', :, :, 0:3, :]
        expected = p4d.ix[:, :, 0:3, :]
        assert_panel_equal(results['L1'], expected['L1'])

        # test a transpose
        # results  = p5d.transpose(1,2,3,4,0)
        # expected =

if __name__ == '__main__':
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)


"""Nearest Neighbor Classification"""

# Authors: Jake Vanderplas <vanderplas@astro.washington.edu>
#          Fabian Pedregosa <fabian.pedregosa@inria.fr>
#          Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Sparseness support by Lars Buitinck <L.J.Buitinck@uva.nl>
#          Multi-output support by Arnaud Joly <a.joly@ulg.ac.be>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam

import numpy as np
from scipy import stats
from ..utils.extmath import weighted_mode

from .base import \
    _check_weights, _get_weights, \
    NeighborsBase, KNeighborsMixin,\
    RadiusNeighborsMixin, SupervisedIntegerMixin
from ..base import ClassifierMixin
from ..utils import check_array

class KNeighborsClassifier(NeighborsBase, KNeighborsMixin,
                           SupervisedIntegerMixin, ClassifierMixin):
    """Classifier implementing the k-nearest neighbors vote.

    Read more in the :ref:`User Guide <classification>`.

    Parameters
    ----------
    n_neighbors : int, optional (default = 5)
        Number of neighbors to use by default for :meth:`k_neighbors` queries.

    weights : str or callable
        weight function used in prediction.  Possible values:

        - 'uniform' : uniform weights.  All points in each neighborhood
          are weighted equally.
        - 'distance' : weight points by the inverse of their distance.
          in this case, closer neighbors of a query point will have a
          greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an
          array of distances, and returns an array of the same shape
          containing the weights.

        Uniform weights are used by default.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        Algorithm used to compute the nearest neighbors:

        - 'ball_tree' will use :class:`BallTree`
        - 'kd_tree' will use :class:`KDTree`
        - 'brute' will use a brute-force search.
        - 'auto' will attempt to decide the most appropriate algorithm
          based on the values passed to :meth:`fit` method.

        Note: fitting on sparse input will override the setting of
        this parameter, using brute force.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or KDTree.  This can affect the
        speed of the construction and query, as well as the memory
        required to store the tree.  The optimal value depends on the
        nature of the problem.

    metric : string or DistanceMetric object (default = 'minkowski')
        the distance metric to use for the tree.  The default metric is
        minkowski, and with p=2 is equivalent to the standard Euclidean
        metric. See the documentation of the DistanceMetric class for a
        list of available metrics.

    p : integer, optional (default = 2)
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params : dict, optional (default = None)
        Additional keyword arguments for the metric function.

    n_jobs : int, optional (default = 1)
        The number of parallel jobs to run for neighbors search.
        If ``-1``, then the number of jobs is set to the number of CPU cores.
        Doesn't affect :meth:`fit` method.

    Examples
    --------
    >>> X = [[0], [1], [2], [3]]
    >>> y = [0, 0, 1, 1]
    >>> from sklearn.neighbors import KNeighborsClassifier
    >>> neigh = KNeighborsClassifier(n_neighbors=3)
    >>> neigh.fit(X, y) # doctest: +ELLIPSIS
    KNeighborsClassifier(...)
    >>> print(neigh.predict([[1.1]]))
    [0]
    >>> print(neigh.predict_proba([[0.9]]))
    [[ 0.66666667  0.33333333]]

    See also
    --------
    RadiusNeighborsClassifier
    KNeighborsRegressor
    RadiusNeighborsRegressor
    NearestNeighbors

    Notes
    -----
    See :ref:`Nearest Neighbors <neighbors>` in the online documentation
    for a discussion of the choice of ``algorithm`` and ``leaf_size``.

    .. warning::

       Regarding the Nearest Neighbors algorithms, if it is found that two
       neighbors, neighbor `k+1` and `k`, have identical distances but
       but different labels, the results will depend on the ordering of the
       training data.

    http://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
    """

    def __init__(self, n_neighbors=5,
                 weights='uniform', algorithm='auto', leaf_size=30,
                 p=2, metric='minkowski', metric_params=None, n_jobs=1,
                 **kwargs):

        self._init_params(n_neighbors=n_neighbors,
                          algorithm=algorithm,
                          leaf_size=leaf_size, metric=metric, p=p,
                          metric_params=metric_params, n_jobs=n_jobs, **kwargs)
        self.weights = _check_weights(weights)

    def predict(self, X):
        """Predict the class labels for the provided data

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        y : array of shape [n_samples] or [n_samples, n_outputs]
            Class labels for each data sample.
        """
        X = check_array(X, accept_sparse='csr')

        neigh_dist, neigh_ind = self.kneighbors(X)

        classes_ = self.classes_
        _y = self._y
        if not self.outputs_2d_:
            _y = self._y.reshape((-1, 1))
            classes_ = [self.classes_]

        n_outputs = len(classes_)
        n_samples = X.shape[0]
        weights = _get_weights(neigh_dist, self.weights)

        y_pred = np.empty((n_samples, n_outputs), dtype=classes_[0].dtype)
        for k, classes_k in enumerate(classes_):
            if weights is None:
                mode, _ = stats.mode(_y[neigh_ind, k], axis=1)
            else:
                mode, _ = weighted_mode(_y[neigh_ind, k], weights, axis=1)

            mode = np.asarray(mode.ravel(), dtype=np.intp)
            y_pred[:, k] = classes_k.take(mode)

        if not self.outputs_2d_:
            y_pred = y_pred.ravel()

        return y_pred

    def predict_proba(self, X):
        """Return probability estimates for the test data X.

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        p : array of shape = [n_samples, n_classes], or a list of n_outputs
            of such arrays if n_outputs > 1.
            The class probabilities of the input samples. Classes are ordered
            by lexicographic order.
        """
        X = check_array(X, accept_sparse='csr')

        neigh_dist, neigh_ind = self.kneighbors(X)

        classes_ = self.classes_
        _y = self._y
        if not self.outputs_2d_:
            _y = self._y.reshape((-1, 1))
            classes_ = [self.classes_]

        n_samples = X.shape[0]

        weights = _get_weights(neigh_dist, self.weights)
        if weights is None:
            weights = np.ones_like(neigh_ind)

        all_rows = np.arange(X.shape[0])
        probabilities = []
        for k, classes_k in enumerate(classes_):
            pred_labels = _y[:, k][neigh_ind]
            proba_k = np.zeros((n_samples, classes_k.size))

            # a simple ':' index doesn't work right
            for i, idx in enumerate(pred_labels.T):  # loop is O(n_neighbors)
                proba_k[all_rows, idx] += weights[:, i]

            # normalize 'votes' into real [0,1] probabilities
            normalizer = proba_k.sum(axis=1)[:, np.newaxis]
            normalizer[normalizer == 0.0] = 1.0
            proba_k /= normalizer

            probabilities.append(proba_k)

        if not self.outputs_2d_:
            probabilities = probabilities[0]

        return probabilities

class RadiusNeighborsClassifier(NeighborsBase, RadiusNeighborsMixin,
                                SupervisedIntegerMixin, ClassifierMixin):
    """Classifier implementing a vote among neighbors within a given radius

    Read more in the :ref:`User Guide <classification>`.

    Parameters
    ----------
    radius : float, optional (default = 1.0)
        Range of parameter space to use by default for :meth`radius_neighbors`
        queries.

    weights : str or callable
        weight function used in prediction.  Possible values:

        - 'uniform' : uniform weights.  All points in each neighborhood
          are weighted equally.
        - 'distance' : weight points by the inverse of their distance.
          in this case, closer neighbors of a query point will have a
          greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an
          array of distances, and returns an array of the same shape
          containing the weights.

        Uniform weights are used by default.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        Algorithm used to compute the nearest neighbors:

        - 'ball_tree' will use :class:`BallTree`
        - 'kd_tree' will use :class:`KDtree`
        - 'brute' will use a brute-force search.
        - 'auto' will attempt to decide the most appropriate algorithm
          based on the values passed to :meth:`fit` method.

        Note: fitting on sparse input will override the setting of
        this parameter, using brute force.

    leaf_size : int, optional (default = 30)
        Leaf size passed to BallTree or KDTree.  This can affect the
        speed of the construction and query, as well as the memory
        required to store the tree.  The optimal value depends on the
        nature of the problem.

    metric : string or DistanceMetric object (default='minkowski')
        the distance metric to use for the tree.  The default metric is
        minkowski, and with p=2 is equivalent to the standard Euclidean
        metric. See the documentation of the DistanceMetric class for a
        list of available metrics.

    p : integer, optional (default = 2)
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    outlier_label : int, optional (default = None)
        Label, which is given for outlier samples (samples with no
        neighbors on given radius).
        If set to None, ValueError is raised, when outlier is detected.

    metric_params : dict, optional (default = None)
        Additional keyword arguments for the metric function.

    Examples
    --------
    >>> X = [[0], [1], [2], [3]]
    >>> y = [0, 0, 1, 1]
    >>> from sklearn.neighbors import RadiusNeighborsClassifier
    >>> neigh = RadiusNeighborsClassifier(radius=1.0)
    >>> neigh.fit(X, y) # doctest: +ELLIPSIS
    RadiusNeighborsClassifier(...)
    >>> print(neigh.predict([[1.5]]))
    [0]

    See also
    --------
    KNeighborsClassifier
    RadiusNeighborsRegressor
    KNeighborsRegressor
    NearestNeighbors

    Notes
    -----
    See :ref:`Nearest Neighbors <neighbors>` in the online documentation
    for a discussion of the choice of ``algorithm`` and ``leaf_size``.

    http://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
    """

    def __init__(self, radius=1.0, weights='uniform',
                 algorithm='auto', leaf_size=30, p=2, metric='minkowski',
                 outlier_label=None, metric_params=None, **kwargs):
        self._init_params(radius=radius,
                          algorithm=algorithm,
                          leaf_size=leaf_size,
                          metric=metric, p=p, metric_params=metric_params,
                          **kwargs)
        self.weights = _check_weights(weights)
        self.outlier_label = outlier_label

    def predict(self, X):
        """Predict the class labels for the provided data

        Parameters
        ----------
        X : array-like, shape (n_query, n_features), \
                or (n_query, n_indexed) if metric == 'precomputed'
            Test samples.

        Returns
        -------
        y : array of shape [n_samples] or [n_samples, n_outputs]
            Class labels for each data sample.

        """
        X = check_array(X, accept_sparse='csr')
        n_samples = X.shape[0]

        neigh_dist, neigh_ind = self.radius_neighbors(X)
        inliers = [i for i, nind in enumerate(neigh_ind) if len(nind) != 0]
        outliers = [i for i, nind in enumerate(neigh_ind) if len(nind) == 0]

        classes_ = self.classes_
        _y = self._y
        if not self.outputs_2d_:
            _y = self._y.reshape((-1, 1))
            classes_ = [self.classes_]
        n_outputs = len(classes_)

        if self.outlier_label is not None:
            neigh_dist[outliers] = 1e-6
        elif outliers:
            raise ValueError('No neighbors found for test samples %r, '
                             'you can try using larger radius, '
                             'give a label for outliers, '
                             'or consider removing them from your dataset.'
                             % outliers)

        weights = _get_weights(neigh_dist, self.weights)

        y_pred = np.empty((n_samples, n_outputs), dtype=classes_[0].dtype)
        for k, classes_k in enumerate(classes_):
            pred_labels = np.array([_y[ind, k] for ind in neigh_ind],
                                   dtype=object)
            if weights is None:
                mode = np.array([stats.mode(pl)[0]
                                 for pl in pred_labels[inliers]], dtype=np.int)
            else:
                mode = np.array([weighted_mode(pl, w)[0]
                                 for (pl, w)
                                 in zip(pred_labels[inliers], weights)],
                                dtype=np.int)

            mode = mode.ravel()

            y_pred[inliers, k] = classes_k.take(mode)

        if outliers:
            y_pred[outliers, :] = self.outlier_label

        if not self.outputs_2d_:
            y_pred = y_pred.ravel()

        return y_pred


"""Fixture module to skip the datasets loading when offline

Mock urllib2 access to mldata.org and create a temporary data folder.
"""

from os import makedirs
from os.path import join
import numpy as np
import tempfile
import shutil

from sklearn import datasets
from sklearn.utils.testing import install_mldata_mock
from sklearn.utils.testing import uninstall_mldata_mock

def globs(globs):
    # Create a temporary folder for the data fetcher
    global custom_data_home
    custom_data_home = tempfile.mkdtemp()
    makedirs(join(custom_data_home, 'mldata'))
    globs['custom_data_home'] = custom_data_home
    return globs

def setup_module():
    # setup mock urllib2 module to avoid downloading from mldata.org
    install_mldata_mock({
        'mnist-original': {
            'data': np.empty((70000, 784)),
            'label': np.repeat(np.arange(10, dtype='d'), 7000),
        },
        'iris': {
            'data': np.empty((150, 4)),
        },
        'datasets-uci-iris': {
            'double0': np.empty((150, 4)),
            'class': np.empty((150,)),
        },
    })

def teardown_module():
    uninstall_mldata_mock()
    shutil.rmtree(custom_data_home)


# Module to read the rather complex config data

import logging
import os
import pprint

from astropy import constants, units as u
import numpy as np
import pandas as pd
import yaml

import tardis
from tardis.io.model_reader import read_density_file, \
    calculate_density_after_time, read_abundances_file
from tardis.io.config_validator import ConfigurationValidator
from tardis import atomic
from tardis.util import species_string_to_tuple, parse_quantity, \
    element_symbol2atomic_number

import copy

pp = pprint.PrettyPrinter(indent=4)

logger = logging.getLogger(__name__)

data_dir = os.path.join(tardis.__path__[0], 'data')

default_config_definition_file = os.path.join(data_dir,
                                              'tardis_config_definition.yml')
#File parsers for different file formats:

density_structure_fileparser = {}

inv_ni56_efolding_time = 1 / (8.8 * u.day)
inv_co56_efolding_time = 1 / (113.7 * u.day)
inv_cr48_efolding_time = 1 / (1.29602 * u.day)
inv_v48_efolding_time = 1 / (23.0442 * u.day)
inv_fe52_efolding_time = 1 / (0.497429 * u.day)
inv_mn52_efolding_time = 1 / (0.0211395 * u.day)

class ConfigurationError(ValueError):
    pass

def parse_quantity_linspace(quantity_linspace_dictionary, add_one=True):
    """
    parse a dictionary of the following kind
    {'start': 5000 km/s,
     'stop': 10000 km/s,
     'num': 1000}

    Parameters
    ----------

    quantity_linspace_dictionary: ~dict

    add_one: boolean, default: True

    Returns
    -------

    ~np.array

    """

    start = parse_quantity(quantity_linspace_dictionary['start'])
    stop = parse_quantity(quantity_linspace_dictionary['stop'])

    try:
        stop = stop.to(start.unit)
    except u.UnitsError:
        raise ConfigurationError('"start" and "stop" keyword must be compatible quantities')

    num = quantity_linspace_dictionary['num']
    if add_one:
        num += 1

    return np.linspace(start.value, stop.value, num=num) * start.unit

def parse_spectral_bin(spectral_bin_boundary_1, spectral_bin_boundary_2):
    spectral_bin_boundary_1 = parse_quantity(spectral_bin_boundary_1).to('Angstrom', u.spectral())
    spectral_bin_boundary_2 = parse_quantity(spectral_bin_boundary_2).to('Angstrom', u.spectral())

    spectrum_start_wavelength = min(spectral_bin_boundary_1, spectral_bin_boundary_2)
    spectrum_end_wavelength = max(spectral_bin_boundary_1, spectral_bin_boundary_2)

    return spectrum_start_wavelength, spectrum_end_wavelength

def calculate_exponential_density(velocities, v_0, rho0):
    """
    This function computes the exponential density profile.
    :math:`\\rho = \\rho_0 \\times \\exp \\left( -\\frac{v}{v_0} \\right)`

    Parameters
    ----------

    velocities : ~astropy.Quantity
        Array like velocity profile
    velocity_0 : ~astropy.Quantity
        reference velocity
    rho0 : ~astropy.Quantity
        reference density

    Returns
    -------

    densities : ~astropy.Quantity

    """
    densities = rho0 * np.exp(-(velocities / v_0))
    return densities

def calculate_power_law_density(velocities, velocity_0, rho_0, exponent):
    """

    This function computes a descret exponential density profile.
    :math:`\\rho = \\rho_0 \\times \\left( \\frac{v}{v_0} \\right)^n`

    Parameters
    ----------

    velocities : ~astropy.Quantity
        Array like velocity profile
    velocity_0 : ~astropy.Quantity
        reference velocity
    rho0 : ~astropy.Quantity
        reference density
    exponent : ~float
        exponent used in the powerlaw

    Returns
    -------

    densities : ~astropy.Quantity

    """
    densities = rho_0 * np.power((velocities / velocity_0), exponent)
    return densities

def parse_model_file_section(model_setup_file_dict, time_explosion):
    def parse_artis_model_setup_files(model_file_section_dict, time_explosion):

        ###### Reading the structure part of the ARTIS file pair
        structure_fname = model_file_section_dict['structure_fname']

        for i, line in enumerate(file(structure_fname)):
            if i == 0:
                no_of_shells = np.int64(line.strip())
            elif i == 1:
                time_of_model = u.Quantity(float(line.strip()), 'day').to('s')
            elif i == 2:
                break

        artis_model_columns = ['velocities', 'mean_densities_0', 'ni56_fraction', 'co56_fraction', 'fe52_fraction',
                               'cr48_fraction']
        artis_model = np.recfromtxt(structure_fname, skip_header=2, usecols=(1, 2, 4, 5, 6, 7), unpack=True,
                                    dtype=[(item, np.float64) for item in artis_model_columns])
        #converting densities from log(g/cm^3) to g/cm^3 and stretching it to the current ti
        velocities = u.Quantity(np.append([0], artis_model['velocities']), 'km/s').to('cm/s')
        mean_densities_0 = u.Quantity(10 ** artis_model['mean_densities_0'], 'g/cm^3')

        mean_densities = calculate_density_after_time(mean_densities_0, time_of_model, time_explosion)

        #Verifying information
        if len(mean_densities) == no_of_shells:
            logger.debug('Verified ARTIS model structure file %s (no_of_shells=length of dataset)', structure_fname)
        else:
            raise ConfigurationError(
                'Error in ARTIS file %s - Number of shells not the same as dataset length' % structure_fname)

        v_inner = velocities[:-1]
        v_outer = velocities[1:]

        volumes = (4 * np.pi / 3) * (time_of_model ** 3) * ( v_outer ** 3 - v_inner ** 3)
        masses = (volumes * mean_densities_0 / constants.M_sun).to(1)

        logger.info('Read ARTIS configuration file %s - found %d zones with total mass %g Msun', structure_fname,
                    no_of_shells, sum(masses.value))

        if 'v_lowest' in model_file_section_dict:
            v_lowest = parse_quantity(model_file_section_dict['v_lowest']).to('cm/s').value
            min_shell = v_inner.value.searchsorted(v_lowest)
        else:
            min_shell = 1

        if 'v_highest' in model_file_section_dict:
            v_highest = parse_quantity(model_file_section_dict['v_highest']).to('cm/s').value
            max_shell = v_outer.value.searchsorted(v_highest)
        else:
            max_shell = no_of_shells
        artis_model = artis_model[min_shell:max_shell]
        v_inner = v_inner[min_shell:max_shell]
        v_outer = v_outer[min_shell:max_shell]
        mean_densities = mean_densities[min_shell:max_shell]

        ###### Reading the abundance part of the ARTIS file pair
        abundances_fname = model_file_section_dict['abundances_fname']
        abundances = pd.DataFrame(np.loadtxt(abundances_fname)[min_shell:max_shell, 1:].transpose(),
                                  index=np.arange(1, 31))

        ni_stable = abundances.ix[28] - artis_model['ni56_fraction']
        co_stable = abundances.ix[27] - artis_model['co56_fraction']
        fe_stable = abundances.ix[26] - artis_model['fe52_fraction']
        mn_stable = abundances.ix[25] - 0.0
        cr_stable = abundances.ix[24] - artis_model['cr48_fraction']
        v_stable = abundances.ix[23] - 0.0
        ti_stable = abundances.ix[22] - 0.0

        abundances.ix[28] = ni_stable
        abundances.ix[28] += artis_model['ni56_fraction'] * np.exp(
            -(time_explosion * inv_ni56_efolding_time).to(1).value)

        abundances.ix[27] = co_stable
        abundances.ix[27] += artis_model['co56_fraction'] * np.exp(
            -(time_explosion * inv_co56_efolding_time).to(1).value)
        abundances.ix[27] += (inv_ni56_efolding_time * artis_model['ni56_fraction'] /
                              (inv_ni56_efolding_time - inv_co56_efolding_time)) * \
                             (np.exp(-(inv_co56_efolding_time * time_explosion).to(1).value) - np.exp(
                                 -(inv_ni56_efolding_time * time_explosion).to(1).value))

        abundances.ix[26] = fe_stable
        abundances.ix[26] += artis_model['fe52_fraction'] * np.exp(
            -(time_explosion * inv_fe52_efolding_time).to(1).value)
        abundances.ix[26] += ((artis_model['co56_fraction'] * inv_ni56_efolding_time
                               - artis_model['co56_fraction'] * inv_co56_efolding_time
                               + artis_model['ni56_fraction'] * inv_ni56_efolding_time
                               - artis_model['ni56_fraction'] * inv_co56_efolding_time
                               - artis_model['co56_fraction'] * inv_ni56_efolding_time * np.exp(
            -(inv_co56_efolding_time * time_explosion).to(1).value)
                               + artis_model['co56_fraction'] * inv_co56_efolding_time * np.exp(
            -(inv_co56_efolding_time * time_explosion).to(1).value)
                               - artis_model['ni56_fraction'] * inv_ni56_efolding_time * np.exp(
            -(inv_co56_efolding_time * time_explosion).to(1).value)
                               + artis_model['ni56_fraction'] * inv_co56_efolding_time * np.exp(
            -(inv_ni56_efolding_time * time_explosion).to(1).value))
                              / (inv_ni56_efolding_time - inv_co56_efolding_time))

        abundances.ix[25] = mn_stable
        abundances.ix[25] += (inv_fe52_efolding_time * artis_model['fe52_fraction'] /
                              (inv_fe52_efolding_time - inv_mn52_efolding_time)) * \
                             (np.exp(-(inv_mn52_efolding_time * time_explosion).to(1).value) - np.exp(
                                 -(inv_fe52_efolding_time * time_explosion).to(1).value))

        abundances.ix[24] = cr_stable
        abundances.ix[24] += artis_model['cr48_fraction'] * np.exp(
            -(time_explosion * inv_cr48_efolding_time).to(1).value)
        abundances.ix[24] += ((artis_model['fe52_fraction'] * inv_fe52_efolding_time
                               - artis_model['fe52_fraction'] * inv_mn52_efolding_time
                               - artis_model['fe52_fraction'] * inv_fe52_efolding_time * np.exp(
            -(inv_mn52_efolding_time * time_explosion).to(1).value)
                               + artis_model['fe52_fraction'] * inv_mn52_efolding_time * np.exp(
            -(inv_fe52_efolding_time * time_explosion).to(1).value))
                              / (inv_fe52_efolding_time - inv_mn52_efolding_time))

        abundances.ix[23] = v_stable
        abundances.ix[23] += (inv_cr48_efolding_time * artis_model['cr48_fraction'] /
                              (inv_cr48_efolding_time - inv_v48_efolding_time)) * \
                             (np.exp(-(inv_v48_efolding_time * time_explosion).to(1).value) - np.exp(
                                 -(inv_cr48_efolding_time * time_explosion).to(1).value))

        abundances.ix[22] = ti_stable
        abundances.ix[22] += ((artis_model['cr48_fraction'] * inv_cr48_efolding_time
                               - artis_model['cr48_fraction'] * inv_v48_efolding_time
                               - artis_model['cr48_fraction'] * inv_cr48_efolding_time * np.exp(
            -(inv_v48_efolding_time * time_explosion).to(1).value)
                               + artis_model['cr48_fraction'] * inv_v48_efolding_time * np.exp(
            -(inv_cr48_efolding_time * time_explosion).to(1).value))
                              / (inv_cr48_efolding_time - inv_v48_efolding_time))

        if 'split_shells' in model_file_section_dict:
            split_shells = int(model_file_section_dict['split_shells'])
        else:
            split_shells = 1

        if split_shells > 1:
            logger.info('Increasing the number of shells by a factor of %s' % split_shells)
            no_of_shells = len(v_inner)
            velocities = np.linspace(v_inner[0], v_outer[-1], no_of_shells * split_shells + 1)
            v_inner = velocities[:-1]
            v_outer = velocities[1:]
            old_mean_densities = mean_densities
            mean_densities = np.empty(no_of_shells * split_shells) * old_mean_densities.unit
            new_abundance_data = np.empty((abundances.values.shape[0], no_of_shells * split_shells))
            for i in xrange(split_shells):
                mean_densities[i::split_shells] = old_mean_densities
                new_abundance_data[:, i::split_shells] = abundances.values

            abundances = pd.DataFrame(new_abundance_data, index=abundances.index)

            #def parser_simple_ascii_model

        return v_inner, v_outer, mean_densities, abundances

    model_file_section_parser = {}
    model_file_section_parser['artis'] = parse_artis_model_setup_files

    try:
        parser = model_file_section_parser[model_setup_file_dict['type']]
    except KeyError:
        raise ConfigurationError('In abundance file section only types %s are allowed (supplied %s) ' %
                                 (model_file_section_parser.keys(), model_file_section_parser['type']))

    return parser(model_setup_file_dict, time_explosion)

def parse_density_file_section(density_file_dict, time_explosion):
    density_file_parser = {}

    def parse_artis_density(density_file_dict, time_explosion):
        density_file = density_file_dict['name']
        for i, line in enumerate(file(density_file)):
            if i == 0:
                no_of_shells = np.int64(line.strip())
            elif i == 1:
                time_of_model = u.Quantity(float(line.strip()), 'day').to('s')
            elif i == 2:
                break

        velocities, mean_densities_0 = np.recfromtxt(density_file, skip_header=2, usecols=(1, 2), unpack=True)
        #converting densities from log(g/cm^3) to g/cm^3 and stretching it to the current ti
        velocities = u.Quantity(np.append([0], velocities), 'km/s').to('cm/s')
        mean_densities_0 = u.Quantity(10 ** mean_densities_0, 'g/cm^3')

        mean_densities = calculate_density_after_time(mean_densities_0, time_of_model, time_explosion)

        #Verifying information
        if len(mean_densities) == no_of_shells:
            logger.debug('Verified ARTIS file %s (no_of_shells=length of dataset)', density_file)
        else:
            raise ConfigurationError(
                'Error in ARTIS file %s - Number of shells not the same as dataset length' % density_file)

        min_shell = 1
        max_shell = no_of_shells

        v_inner = velocities[:-1]
        v_outer = velocities[1:]

        volumes = (4 * np.pi / 3) * (time_of_model ** 3) * ( v_outer ** 3 - v_inner ** 3)
        masses = (volumes * mean_densities_0 / constants.M_sun).to(1)

        logger.info('Read ARTIS configuration file %s - found %d zones with total mass %g Msun', density_file,
                    no_of_shells, sum(masses.value))

        if 'v_lowest' in density_file_dict:
            v_lowest = parse_quantity(density_file_dict['v_lowest']).to('cm/s').value
            min_shell = v_inner.value.searchsorted(v_lowest)
        else:
            min_shell = 1

        if 'v_highest' in density_file_dict:
            v_highest = parse_quantity(density_file_dict['v_highest']).to('cm/s').value
            max_shell = v_outer.value.searchsorted(v_highest)
        else:
            max_shell = no_of_shells

        v_inner = v_inner[min_shell:max_shell]
        v_outer = v_outer[min_shell:max_shell]
        mean_densities = mean_densities[min_shell:max_shell]

        return v_inner, v_outer, mean_densities, min_shell, max_shell

    density_file_parser['artis'] = parse_artis_density

    try:
        parser = density_file_parser[density_file_dict['type']]
    except KeyError:
        raise ConfigurationError('In abundance file section only types %s are allowed (supplied %s) ' %
                                 (density_file_parser.keys(), density_file_dict['type']))

    return parser(density_file_dict, time_explosion)

def parse_density_section(density_dict, v_inner, v_outer, time_explosion):
    density_parser = {}

    #Parse density uniform
    def parse_uniform(density_dict, v_inner, v_outer, time_explosion):
        no_of_shells = len(v_inner)
        return density_dict['value'].to('g cm^-3') * np.ones(no_of_shells)

    density_parser['uniform'] = parse_uniform

    #Parse density branch85 w7
    def parse_branch85(density_dict, v_inner, v_outer, time_explosion):
        velocities = 0.5 * (v_inner + v_outer)

        densities = calculate_power_law_density(velocities,
                                                density_dict['w7_v_0'],
                                                density_dict['w7_rho_0'], -7)

        densities = calculate_density_after_time(densities,
                                                 density_dict['w7_time_0'],
                                                 time_explosion)

        return densities

    density_parser['branch85_w7'] = parse_branch85

    def parse_power_law(density_dict, v_inner, v_outer, time_explosion):
        time_0 = density_dict.pop('time_0')
        rho_0 = density_dict.pop('rho_0')
        v_0 = density_dict.pop('v_0')
        exponent = density_dict.pop('exponent')
        velocities = 0.5 * (v_inner + v_outer)
        densities = calculate_power_law_density(velocities, v_0, rho_0, exponent)
        densities = calculate_density_after_time(densities, time_0, time_explosion)
        return densities

    density_parser['power_law'] = parse_power_law

    def parse_exponential(density_dict, v_inner, v_outer, time_explosion):
        time_0 = density_dict.pop('time_0')
        rho_0 = density_dict.pop('rho_0')
        v_0 = density_dict.pop('v_0')

        velocities = 0.5 * (v_inner + v_outer)
        densities = calculate_exponential_density(velocities, v_0, rho_0)
        densities = calculate_density_after_time(densities, time_0, time_explosion)
        return densities

    density_parser['exponential'] = parse_exponential

    try:
        parser = density_parser[density_dict['type']]
    except KeyError:
        raise ConfigurationError('In density section only types %s are allowed (supplied %s) ' %
                                 (density_parser.keys(), density_dict['type']))
    return parser(density_dict, v_inner, v_outer, time_explosion)

def parse_abundance_file_section(abundance_file_dict, abundances, min_shell, max_shell):
    abundance_file_parser = {}

    def parse_artis(abundance_file_dict, abundances, min_shell, max_shell):
        #### ---- debug ----
        time_of_model = 0.0

        ####
        fname = abundance_file_dict['name']
        max_atom = 30
        logger.info("Parsing ARTIS Abundance section from shell %d to %d", min_shell, max_shell)

        abundances.values[:max_atom, :] = np.loadtxt(fname)[min_shell:max_shell, 1:].transpose()

        return abundances

    abundance_file_parser['artis'] = parse_artis

    try:
        parser = abundance_file_parser[abundance_file_dict['type']]
    except KeyError:
        raise ConfigurationError('In abundance file section only types %s are allowed (supplied %s) ' %
                                 (abundance_file_parser.keys(), abundance_file_dict['type']))

    return parser(abundance_file_dict, abundances, min_shell, max_shell)

def parse_supernova_section(supernova_dict):
    """
    Parse the supernova section

    Parameters
    ----------

    supernova_dict: dict
        YAML parsed supernova dict

    Returns
    -------

    config_dict: dict

    """
    config_dict = {}

    #parse luminosity
    luminosity_value, luminosity_unit = supernova_dict['luminosity_requested'].strip().split()

    if luminosity_unit == 'log_lsun':
        config_dict['luminosity_requested'] = 10 ** (
        float(luminosity_value) + np.log10(constants.L_sun.cgs.value)) * u.erg / u.s
    else:
        config_dict['luminosity_requested'] = (float(luminosity_value) * u.Unit(luminosity_unit)).to('erg/s')

    config_dict['time_explosion'] = parse_quantity(supernova_dict['time_explosion']).to('s')

    if 'distance' in supernova_dict:
        config_dict['distance'] = parse_quantity(supernova_dict['distance'])
    else:
        config_dict['distance'] = None

    if 'luminosity_wavelength_start' in supernova_dict:
        config_dict['luminosity_nu_end'] = parse_quantity(supernova_dict['luminosity_wavelength_start']). \
            to('Hz', u.spectral())
    else:
        config_dict['luminosity_nu_end'] = np.inf * u.Hz

    if 'luminosity_wavelength_end' in supernova_dict:
        config_dict['luminosity_nu_start'] = parse_quantity(supernova_dict['luminosity_wavelength_end']). \
            to('Hz', u.spectral())
    else:
        config_dict['luminosity_nu_start'] = 0.0 * u.Hz

    return config_dict

def parse_spectrum_list2dict(spectrum_list):
    """
    Parse the spectrum list [start, stop, num] to a list
    """

    if spectrum_list[0].unit.physical_type != 'length' and \
                    spectrum_list[1].unit.physical_type != 'length':
        raise ValueError('start and end of spectrum need to be a length')

    spectrum_config_dict = {}
    spectrum_config_dict['start'] = spectrum_list[0]
    spectrum_config_dict['end'] = spectrum_list[1]
    spectrum_config_dict['bins'] = spectrum_list[2]

    spectrum_frequency = np.linspace(
        spectrum_config_dict['end'].to('Hz', u.spectral()),
        spectrum_config_dict['start'].to('Hz', u.spectral()),
        num=spectrum_config_dict['bins'] + 1)

    spectrum_config_dict['frequency'] = spectrum_frequency

    return spectrum_config_dict

def parse_convergence_section(convergence_section_dict):
    """
    Parse the convergence section dictionary

    Parameters
    ----------

    convergence_section_dict: ~dict
        dictionary
    """

    convergence_parameters = ['damping_constant', 'threshold', 'fraction',
            'hold_iterations']

    for convergence_variable in ['t_inner', 't_rad', 'w']:
        if convergence_variable not in convergence_section_dict:
            convergence_section_dict[convergence_variable] = {}
        convergence_variable_section = convergence_section_dict[convergence_variable]
        for param in convergence_parameters:
            if convergence_variable_section.get(param, None) is None:
                if param in convergence_section_dict:
                    convergence_section_dict[convergence_variable][param] = (
                        convergence_section_dict[param])

    return convergence_section_dict

def calculate_w7_branch85_densities(velocities, time_explosion, time_0=19.9999584, density_coefficient=3e29):
    """
        Generated densities from the fit to W7 in Branch 85 page 620 (citation missing)

        Parameters
        ----------

        velocities : `~numpy.ndarray`
            velocities in cm/s

        time_explosion : `float`
            time since explosion needed to descale density with expansion

        time_0 : `float`
            time in seconds of the w7 model - default 19.999, no reason to change

        density_coefficient : `float`
            coefficient for the polynomial - obtained by fitting to W7, no reason to change

    """
    densities = density_coefficient * (velocities * 1e-5) ** -7
    densities = calculate_density_after_time(densities, time_0, time_explosion)

    return densities[1:]

class ConfigurationNameSpace(dict):
    """
    The configuration name space class allows to wrap a dictionary and adds
    utility functions for easy access. Accesses like a.b.c are then possible

    Code from http://goo.gl/KIaq8I

    Parameters
    ----------

    config_dict: ~dict
        configuration dictionary

    Returns
    -------

    config_ns: ConfigurationNameSpace
    """

    @classmethod
    def from_yaml(cls, fname):
        """
        Read a configuration from a YAML file

        Parameters
        ----------

        fname: str
            filename or path
        """
        try:
            yaml_dict = yaml.load(file(fname))
        except IOError as e:
            logger.critical('No config file named: %s', fname)
            raise e

        return cls.from_config_dict(yaml_dict)

    @classmethod
    def from_config_dict(cls, config_dict, config_definition_file=None):
        """
        Validating a config file.

        Parameters
        ----------

        config_dict : ~dict
            dictionary of a raw unvalidated config file

        Returns
        -------

        `tardis.config_reader.Configuration`

        """

        if config_definition_file is None:
            config_definition_file = default_config_definition_file

        config_definition = yaml.load(open(config_definition_file))

        return cls(ConfigurationValidator(config_definition,
                                       config_dict).get_config())

    marker = object()
    def __init__(self, value=None):
        if value is None:
            pass
        elif isinstance(value, dict):
            for key in value:
                self.__setitem__(key, value[key])
        else:
            raise TypeError, 'expected dict'

    def __setitem__(self, key, value):
        if isinstance(value, dict) and not isinstance(value,
                                                      ConfigurationNameSpace):
            value = ConfigurationNameSpace(value)

        if key in self and hasattr(self[key], 'unit'):
            value = u.Quantity(value, self[key].unit)

        dict.__setitem__(self, key, value)

    def __getitem__(self, key):
        return super(ConfigurationNameSpace, self).__getitem__(key)

    def __getattr__(self, item):
        if item in self:
            return self[item]
        else:
            super(ConfigurationNameSpace, self).__getattribute__(item)

    __setattr__ = __setitem__

    def __dir__(self):
        return self.keys()

    def get_config_item(self, config_item_string):
        """
        Get configuration items using a string of type 'a.b.param'

        Parameters
        ----------

        config_item_string: ~str
            string of shape 'section1.sectionb.param1'
        """
        config_item_path = config_item_string.split('.')

        if len(config_item_path) == 1:
            config_item = config_item_path[0]

            if config_item.startswith('item'):
                return self[config_item_path[0]]
            else:
                return self[config_item]
        elif len(config_item_path) == 2 and\
                config_item_path[1].startswith('item'):
            return self[config_item_path[0]][
                int(config_item_path[1].replace('item', ''))]

        else:
            return self[config_item_path[0]].get_config_item(
                '.'.join(config_item_path[1:]))

    def set_config_item(self, config_item_string, value):
        """
        set configuration items using a string of type 'a.b.param'

        Parameters
        ----------

        config_item_string: ~str
            string of shape 'section1.sectionb.param1'

        value:
            value to set the parameter with it
        """

        config_item_path = config_item_string.split('.')
        if len(config_item_path) == 1:
            self[config_item_path[0]] = value
        elif len(config_item_path) == 2 and \
                config_item_path[1].startswith('item'):
            current_value = self[config_item_path[0]][
                int(config_item_path[1].replace('item', ''))]
            if hasattr(current_value, 'unit'):
                self[config_item_path[0]][
                    int(config_item_path[1].replace('item', ''))] =\
                    u.Quantity(value, current_value.unit)
            else:
                self[config_item_path[0]][
                    int(config_item_path[1].replace('item', ''))] = value

        else:
            self[config_item_path[0]].set_config_item(
                '.'.join(config_item_path[1:]), value)

    def deepcopy(self):
        return ConfigurationNameSpace(copy.deepcopy(dict(self)))

class Configuration(ConfigurationNameSpace):
    """
    Tardis configuration class
    """

    @classmethod
    def from_yaml(cls, fname, test_parser=False):
        try:
            yaml_dict = yaml.load(open(fname))
        except IOError as e:
            logger.critical('No config file named: %s', fname)
            raise e

        tardis_config_version = yaml_dict.get('tardis_config_version', None)
        if tardis_config_version != 'v1.0':
            raise ConfigurationError('Currently only tardis_config_version v1.0 supported')

        return cls.from_config_dict(yaml_dict, test_parser=test_parser)

    @classmethod
    def from_config_dict(cls, config_dict, atom_data=None, test_parser=False,
                         config_definition_file=None, validate=True):
        """
        Validating and subsequently parsing a config file.

        Parameters
        ----------

        config_dict : ~dict
            dictionary of a raw unvalidated config file

        atom_data: ~tardis.atomic.AtomData
            atom data object. if `None` will be tried to be read from
            atom data file path in the config_dict [default=None]

        test_parser: ~bool
            switch on to ignore a working atom_data, mainly useful for
            testing this reader

        config_definition_file: ~str
            path to config definition file, if `None` will be set to the default
            in the `data` directory that ships with TARDIS

        validate: ~bool
            Turn validation on or off.

        Returns
        -------

        `tardis.config_reader.Configuration`

        """

        if config_definition_file is None:
            config_definition_file = default_config_definition_file

        config_definition = yaml.load(open(config_definition_file))
        if validate:
            validated_config_dict = ConfigurationValidator(config_definition,
                                       config_dict).get_config()
        else:
            validated_config_dict = config_dict

        #First let's see if we can find an atom_db anywhere:
        if test_parser:
            atom_data = None
        elif 'atom_data' in validated_config_dict.keys():
            atom_data_fname = validated_config_dict['atom_data']
            validated_config_dict['atom_data_fname'] = atom_data_fname
        else:
            raise ConfigurationError('No atom_data key found in config or command line')

        if atom_data is None and not test_parser:
            logger.info('Reading Atomic Data from %s', atom_data_fname)
            atom_data = atomic.AtomData.from_hdf5(atom_data_fname)
        else:
            atom_data = atom_data

        #Parsing supernova dictionary
        validated_config_dict['supernova']['luminosity_nu_start'] = \
            validated_config_dict['supernova']['luminosity_wavelength_end'].to(
                u.Hz, u.spectral())
        try:
            validated_config_dict['supernova']['luminosity_nu_end'] = \
                (validated_config_dict['supernova']
                 ['luminosity_wavelength_start'].to(u.Hz, u.spectral()))
        except ZeroDivisionError:
            validated_config_dict['supernova']['luminosity_nu_end'] = (
                np.inf * u.Hz)

        validated_config_dict['supernova']['time_explosion'] = (
            validated_config_dict['supernova']['time_explosion'].cgs)

        validated_config_dict['supernova']['luminosity_requested'] = (
            validated_config_dict['supernova']['luminosity_requested'].cgs)

        #Parsing the model section

        model_section = validated_config_dict['model']
        v_inner = None
        v_outer = None
        mean_densities = None
        abundances = None

        structure_section = model_section['structure']

        if structure_section['type'] == 'specific':
            start, stop, num = model_section['structure']['velocity']
            num += 1
            velocities = np.linspace(start, stop, num)

            v_inner, v_outer = velocities[:-1], velocities[1:]
            mean_densities = parse_density_section(
                model_section['structure']['density'], v_inner, v_outer,
                validated_config_dict['supernova']['time_explosion']).cgs

        elif structure_section['type'] == 'file':
            v_inner, v_outer, mean_densities, inner_boundary_index, \
            outer_boundary_index = read_density_file(
                structure_section['filename'], structure_section['filetype'],
                validated_config_dict['supernova']['time_explosion'],
                structure_section['v_inner_boundary'],
                structure_section['v_outer_boundary'])

        r_inner = validated_config_dict['supernova']['time_explosion'] * v_inner
        r_outer = validated_config_dict['supernova']['time_explosion'] * v_outer
        r_middle = 0.5 * (r_inner + r_outer)

        structure_validated_config_dict = {}
        structure_section['v_inner'] = v_inner.cgs
        structure_section['v_outer'] = v_outer.cgs
        structure_section['mean_densities'] = mean_densities.cgs
        no_of_shells = len(v_inner)
        structure_section['no_of_shells'] = no_of_shells
        structure_section['r_inner'] = r_inner.cgs
        structure_section['r_outer'] = r_outer.cgs
        structure_section['r_middle'] = r_middle.cgs
        structure_section['volumes'] = ((4. / 3) * np.pi * \
                                       (r_outer ** 3 -
                                        r_inner ** 3)).cgs

        #### TODO the following is legacy code and should be removed
        validated_config_dict['structure'] = \
            validated_config_dict['model']['structure']
        # ^^^^^^^^^^^^^^^^

        abundances_section = model_section['abundances']

        if abundances_section['type'] == 'uniform':
            abundances = pd.DataFrame(columns=np.arange(no_of_shells),
                                      index=pd.Index(np.arange(1, 120), name='atomic_number'), dtype=np.float64)

            for element_symbol_string in abundances_section:
                if element_symbol_string == 'type': continue
                z = element_symbol2atomic_number(element_symbol_string)
                abundances.ix[z] = float(abundances_section[element_symbol_string])

        elif abundances_section['type'] == 'file':
            index, abundances = read_abundances_file(abundances_section['filename'], abundances_section['filetype'],
                                                     inner_boundary_index, outer_boundary_index)
            if len(index) != no_of_shells:
                raise ConfigurationError('The abundance file specified has not the same number of cells'
                                         'as the specified density profile')

        abundances = abundances.replace(np.nan, 0.0)

        abundances = abundances[abundances.sum(axis=1) > 0]

        norm_factor = abundances.sum(axis=0)

        if np.any(np.abs(norm_factor - 1) > 1e-12):
            logger.warning("Abundances have not been normalized to 1. - normalizing")
            abundances /= norm_factor

        validated_config_dict['abundances'] = abundances

        ########### DOING PLASMA SECTION ###############
        plasma_section = validated_config_dict['plasma']

        if plasma_section['initial_t_inner'] < 0.0 * u.K:
            luminosity_requested = validated_config_dict['supernova']['luminosity_requested']
            plasma_section['t_inner'] = ((luminosity_requested /
                                          (4 * np.pi * r_inner[0] ** 2 *
                                           constants.sigma_sb)) ** .25).to('K')
            logger.info('"initial_t_inner" is not specified in the plasma '
                        'section - initializing to %s with given luminosity',
                        plasma_section['t_inner'])
        else:
            plasma_section['t_inner'] = plasma_section['initial_t_inner']

        plasma_section['t_rads'] = np.ones(no_of_shells) * \
                                   plasma_section['initial_t_rad']
        if plasma_section['disable_electron_scattering'] is False:
            logger.debug("Electron scattering switched on")
            validated_config_dict['montecarlo']['sigma_thomson'] = 6.652486e-25 / (u.cm ** 2)
        else:
            logger.warn('Disabling electron scattering - this is not physical')
            validated_config_dict['montecarlo']['sigma_thomson'] = 1e-200 / (u.cm ** 2)

        ##### NLTE subsection of Plasma start
        nlte_validated_config_dict = {}
        nlte_species = []
        nlte_section = plasma_section['nlte']

        nlte_species_list = nlte_section.pop('species')
        for species_string in nlte_species_list:
            nlte_species.append(species_string_to_tuple(species_string))

        nlte_validated_config_dict['species'] = nlte_species
        nlte_validated_config_dict['species_string'] = nlte_species_list
        nlte_validated_config_dict.update(nlte_section)

        if 'coronal_approximation' not in nlte_section:
            logger.debug('NLTE "coronal_approximation" not specified in NLTE section - defaulting to False')
            nlte_validated_config_dict['coronal_approximation'] = False

        if 'classical_nebular' not in nlte_section:
            logger.debug('NLTE "classical_nebular" not specified in NLTE section - defaulting to False')
            nlte_validated_config_dict['classical_nebular'] = False

        elif nlte_section:  #checks that the dictionary is not empty
            logger.warn('No "species" given - ignoring other NLTE options given:\n%s',
                        pp.pformat(nlte_section))

        if not nlte_validated_config_dict:
            nlte_validated_config_dict['species'] = []

        plasma_section['nlte'] = nlte_validated_config_dict

        #^^^^^^^^^^^^^^ End of Plasma Section

        ##### Monte Carlo Section

        montecarlo_section = validated_config_dict['montecarlo']
        if montecarlo_section['last_no_of_packets'] < 0:
            montecarlo_section['last_no_of_packets'] = \
                montecarlo_section['no_of_packets']

        default_convergence_section = {'type': 'damped',
                                      'lock_t_inner_cycles': 1,
                                      't_inner_update_exponent': -0.5,
                                      'damping_constant': 0.5}

        if montecarlo_section['convergence_strategy'] is None:
            logger.warning('No convergence criteria selected - '
                           'just damping by 0.5 for w, t_rad and t_inner')
            montecarlo_section['convergence_strategy'] = (
                parse_convergence_section(default_convergence_section))
        else:
            montecarlo_section['convergence_strategy'] = (
                parse_convergence_section(
                    montecarlo_section['convergence_strategy']))

        black_body_section = montecarlo_section['black_body_sampling']
        montecarlo_section['black_body_sampling'] = {}
        montecarlo_section['black_body_sampling']['start'] = \
            black_body_section[0]
        montecarlo_section['black_body_sampling']['end'] = \
            black_body_section[1]
        montecarlo_section['black_body_sampling']['samples'] = \
            black_body_section[2]
        ###### END of convergence section reading

        validated_config_dict['spectrum'] = parse_spectrum_list2dict(
            validated_config_dict['spectrum'])

        return cls(validated_config_dict, atom_data)

    def __init__(self, config_dict, atom_data):
        super(Configuration, self).__init__(config_dict)
        self.atom_data = atom_data
        selected_atomic_numbers = self.abundances.index
        if atom_data is not None:
            self.number_densities = (self.abundances * self.structure.mean_densities.to('g/cm^3').value)
            self.number_densities = self.number_densities.div(self.atom_data.atom_data.mass.ix[selected_atomic_numbers],
                                                              axis=0)
        else:
            logger.critical('atom_data is None, only sensible for testing the parser')


#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
import warnings

from pyspark import since, _NoValue
from pyspark.sql.session import _monkey_patch_RDD, SparkSession
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.readwriter import DataFrameReader
from pyspark.sql.streaming import DataStreamReader
from pyspark.sql.udf import UDFRegistration  # noqa: F401
from pyspark.sql.utils import install_exception_handler

__all__ = ["SQLContext", "HiveContext"]

class SQLContext(object):
    """The entry point for working with structured data (rows and columns) in Spark, in Spark 1.x.

    As of Spark 2.0, this is replaced by :class:`SparkSession`. However, we are keeping the class
    here for backward compatibility.

    A SQLContext can be used create :class:`DataFrame`, register :class:`DataFrame` as
    tables, execute SQL over tables, cache tables, and read parquet files.

    .. deprecated:: 3.0.0
        Use :func:`SparkSession.builder.getOrCreate()` instead.

    Parameters
    ----------
    sparkContext : :class:`SparkContext`
        The :class:`SparkContext` backing this SQLContext.
    sparkSession : :class:`SparkSession`
        The :class:`SparkSession` around which this SQLContext wraps.
    jsqlContext : optional
        An optional JVM Scala SQLContext. If set, we do not instantiate a new
        SQLContext in the JVM, instead we make all calls to this object.
        This is only for internal.

    Examples
    --------
    >>> from datetime import datetime
    >>> from pyspark.sql import Row
    >>> sqlContext = SQLContext(sc)
    >>> allTypes = sc.parallelize([Row(i=1, s="string", d=1.0, l=1,
    ...     b=True, list=[1, 2, 3], dict={"s": 0}, row=Row(a=1),
    ...     time=datetime(2014, 8, 1, 14, 1, 5))])
    >>> df = allTypes.toDF()
    >>> df.createOrReplaceTempView("allTypes")
    >>> sqlContext.sql('select i+1, d+1, not b, list[1], dict["s"], time, row.a '
    ...            'from allTypes where b and i > 0').collect()
    [Row((i + 1)=2, (d + 1)=2.0, (NOT b)=False, list[1]=2, \
        dict[s]=0, time=datetime.datetime(2014, 8, 1, 14, 1, 5), a=1)]
    >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
    [(1, 'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
    """

    _instantiatedContext = None

    def __init__(self, sparkContext, sparkSession=None, jsqlContext=None):
        if sparkSession is None:
            warnings.warn(
                "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
                FutureWarning
            )

        self._sc = sparkContext
        self._jsc = self._sc._jsc
        self._jvm = self._sc._jvm
        if sparkSession is None:
            sparkSession = SparkSession.builder.getOrCreate()
        if jsqlContext is None:
            jsqlContext = sparkSession._jwrapped
        self.sparkSession = sparkSession
        self._jsqlContext = jsqlContext
        _monkey_patch_RDD(self.sparkSession)
        install_exception_handler()
        if (SQLContext._instantiatedContext is None
                or SQLContext._instantiatedContext._sc._jsc is None):
            SQLContext._instantiatedContext = self

    @property
    def _ssql_ctx(self):
        """Accessor for the JVM Spark SQL context.

        Subclasses can override this property to provide their own
        JVM Contexts.
        """
        return self._jsqlContext

    @property
    def _conf(self):
        """Accessor for the JVM SQL-specific configurations"""
        return self.sparkSession._jsparkSession.sessionState().conf()

    @classmethod
    def getOrCreate(cls, sc):
        """
        Get the existing SQLContext or create a new one with given SparkContext.

        .. versionadded:: 1.6.0

        .. deprecated:: 3.0.0
            Use :func:`SparkSession.builder.getOrCreate()` instead.

        Parameters
        ----------
        sc : :class:`SparkContext`
        """
        warnings.warn(
            "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
            FutureWarning
        )

        if (cls._instantiatedContext is None
                or SQLContext._instantiatedContext._sc._jsc is None):
            jsqlContext = sc._jvm.SparkSession.builder().sparkContext(
                sc._jsc.sc()).getOrCreate().sqlContext()
            sparkSession = SparkSession(sc, jsqlContext.sparkSession())
            cls(sc, sparkSession, jsqlContext)
        return cls._instantiatedContext

    def newSession(self):
        """
        Returns a new SQLContext as new session, that has separate SQLConf,
        registered temporary views and UDFs, but shared SparkContext and
        table cache.

        .. versionadded:: 1.6.0
        """
        return self.__class__(self._sc, self.sparkSession.newSession())

    def setConf(self, key, value):
        """Sets the given Spark SQL configuration property.

        .. versionadded:: 1.3.0
        """
        self.sparkSession.conf.set(key, value)

    def getConf(self, key, defaultValue=_NoValue):
        """Returns the value of Spark SQL configuration property for the given key.

        If the key is not set and defaultValue is set, return
        defaultValue. If the key is not set and defaultValue is not set, return
        the system default value.

        .. versionadded:: 1.3.0

        Examples
        --------
        >>> sqlContext.getConf("spark.sql.shuffle.partitions")
        '200'
        >>> sqlContext.getConf("spark.sql.shuffle.partitions", "10")
        '10'
        >>> sqlContext.setConf("spark.sql.shuffle.partitions", "50")
        >>> sqlContext.getConf("spark.sql.shuffle.partitions", "10")
        '50'
        """
        return self.sparkSession.conf.get(key, defaultValue)

    @property
    def udf(self):
        """Returns a :class:`UDFRegistration` for UDF registration.

        .. versionadded:: 1.3.1

        Returns
        -------
        :class:`UDFRegistration`
        """
        return self.sparkSession.udf

    def range(self, start, end=None, step=1, numPartitions=None):
        """
        Create a :class:`DataFrame` with single :class:`pyspark.sql.types.LongType` column named
        ``id``, containing elements in a range from ``start`` to ``end`` (exclusive) with
        step value ``step``.

        .. versionadded:: 1.4.0

        Parameters
        ----------
        start : int
            the start value
        end : int, optional
            the end value (exclusive)
        step : int, optional
            the incremental step (default: 1)
        numPartitions : int, optional
            the number of partitions of the DataFrame

        Returns
        -------
        :class:`DataFrame`

        Examples
        --------
        >>> sqlContext.range(1, 7, 2).collect()
        [Row(id=1), Row(id=3), Row(id=5)]

        If only one argument is specified, it will be used as the end value.

        >>> sqlContext.range(3).collect()
        [Row(id=0), Row(id=1), Row(id=2)]
        """
        return self.sparkSession.range(start, end, step, numPartitions)

    def registerFunction(self, name, f, returnType=None):
        """An alias for :func:`spark.udf.register`.
        See :meth:`pyspark.sql.UDFRegistration.register`.

        .. versionadded:: 1.2.0

        .. deprecated:: 2.3.0
            Use :func:`spark.udf.register` instead.
        """
        warnings.warn(
            "Deprecated in 2.3.0. Use spark.udf.register instead.",
            FutureWarning
        )
        return self.sparkSession.udf.register(name, f, returnType)

    def registerJavaFunction(self, name, javaClassName, returnType=None):
        """An alias for :func:`spark.udf.registerJavaFunction`.
        See :meth:`pyspark.sql.UDFRegistration.registerJavaFunction`.

        .. versionadded:: 2.1.0

        .. deprecated:: 2.3.0
            Use :func:`spark.udf.registerJavaFunction` instead.
        """
        warnings.warn(
            "Deprecated in 2.3.0. Use spark.udf.registerJavaFunction instead.",
            FutureWarning
        )
        return self.sparkSession.udf.registerJavaFunction(name, javaClassName, returnType)

    # TODO(andrew): delete this once we refactor things to take in SparkSession
    def _inferSchema(self, rdd, samplingRatio=None):
        """
        Infer schema from an RDD of Row or tuple.

        Parameters
        ----------
        rdd : :class:`RDD`
            an RDD of Row or tuple
        samplingRatio : float, optional
            sampling ratio, or no sampling (default)

        Returns
        -------
        :class:`pyspark.sql.types.StructType`
        """
        return self.sparkSession._inferSchema(rdd, samplingRatio)

    def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=True):
        """
        Creates a :class:`DataFrame` from an :class:`RDD`, a list or a :class:`pandas.DataFrame`.

        When ``schema`` is a list of column names, the type of each column
        will be inferred from ``data``.

        When ``schema`` is ``None``, it will try to infer the schema (column names and types)
        from ``data``, which should be an RDD of :class:`Row`,
        or :class:`namedtuple`, or :class:`dict`.

        When ``schema`` is :class:`pyspark.sql.types.DataType` or a datatype string it must match
        the real data, or an exception will be thrown at runtime. If the given schema is not
        :class:`pyspark.sql.types.StructType`, it will be wrapped into a
        :class:`pyspark.sql.types.StructType` as its only field, and the field name will be "value",
        each record will also be wrapped into a tuple, which can be converted to row later.

        If schema inference is needed, ``samplingRatio`` is used to determined the ratio of
        rows used for schema inference. The first row will be used if ``samplingRatio`` is ``None``.

        .. versionadded:: 1.3.0

        .. versionchanged:: 2.0.0
           The ``schema`` parameter can be a :class:`pyspark.sql.types.DataType` or a
           datatype string after 2.0.
           If it's not a :class:`pyspark.sql.types.StructType`, it will be wrapped into a
           :class:`pyspark.sql.types.StructType` and each record will also be wrapped into a tuple.

        .. versionchanged:: 2.1.0
           Added verifySchema.

        Parameters
        ----------
        data : :class:`RDD` or iterable
            an RDD of any kind of SQL data representation (:class:`Row`,
            :class:`tuple`, ``int``, ``boolean``, etc.), or :class:`list`, or
            :class:`pandas.DataFrame`.
        schema : :class:`pyspark.sql.types.DataType`, str or list, optional
            a :class:`pyspark.sql.types.DataType` or a datatype string or a list of
            column names, default is None.  The data type string format equals to
            :class:`pyspark.sql.types.DataType.simpleString`, except that top level struct type can
            omit the ``struct<>`` and atomic types use ``typeName()`` as their format, e.g. use
            ``byte`` instead of ``tinyint`` for :class:`pyspark.sql.types.ByteType`.
            We can also use ``int`` as a short name for :class:`pyspark.sql.types.IntegerType`.
        samplingRatio : float, optional
            the sample ratio of rows used for inferring
        verifySchema : bool, optional
            verify data types of every row against schema. Enabled by default.

        Returns
        -------
        :class:`DataFrame`

        Examples
        --------
        >>> l = [('Alice', 1)]
        >>> sqlContext.createDataFrame(l).collect()
        [Row(_1='Alice', _2=1)]
        >>> sqlContext.createDataFrame(l, ['name', 'age']).collect()
        [Row(name='Alice', age=1)]

        >>> d = [{'name': 'Alice', 'age': 1}]
        >>> sqlContext.createDataFrame(d).collect()
        [Row(age=1, name='Alice')]

        >>> rdd = sc.parallelize(l)
        >>> sqlContext.createDataFrame(rdd).collect()
        [Row(_1='Alice', _2=1)]
        >>> df = sqlContext.createDataFrame(rdd, ['name', 'age'])
        >>> df.collect()
        [Row(name='Alice', age=1)]

        >>> from pyspark.sql import Row
        >>> Person = Row('name', 'age')
        >>> person = rdd.map(lambda r: Person(*r))
        >>> df2 = sqlContext.createDataFrame(person)
        >>> df2.collect()
        [Row(name='Alice', age=1)]

        >>> from pyspark.sql.types import *
        >>> schema = StructType([
        ...    StructField("name", StringType(), True),
        ...    StructField("age", IntegerType(), True)])
        >>> df3 = sqlContext.createDataFrame(rdd, schema)
        >>> df3.collect()
        [Row(name='Alice', age=1)]

        >>> sqlContext.createDataFrame(df.toPandas()).collect()  # doctest: +SKIP
        [Row(name='Alice', age=1)]
        >>> sqlContext.createDataFrame(pandas.DataFrame([[1, 2]])).collect()  # doctest: +SKIP
        [Row(0=1, 1=2)]

        >>> sqlContext.createDataFrame(rdd, "a: string, b: int").collect()
        [Row(a='Alice', b=1)]
        >>> rdd = rdd.map(lambda row: row[1])
        >>> sqlContext.createDataFrame(rdd, "int").collect()
        [Row(value=1)]
        >>> sqlContext.createDataFrame(rdd, "boolean").collect() # doctest: +IGNORE_EXCEPTION_DETAIL
        Traceback (most recent call last):
            ...
        Py4JJavaError: ...
        """
        return self.sparkSession.createDataFrame(data, schema, samplingRatio, verifySchema)

    def registerDataFrameAsTable(self, df, tableName):
        """Registers the given :class:`DataFrame` as a temporary table in the catalog.

        Temporary tables exist only during the lifetime of this instance of :class:`SQLContext`.

        .. versionadded:: 1.3.0

        Examples
        --------
        >>> sqlContext.registerDataFrameAsTable(df, "table1")
        """
        df.createOrReplaceTempView(tableName)

    def dropTempTable(self, tableName):
        """ Remove the temporary table from catalog.

        .. versionadded:: 1.6.0

        Examples
        --------
        >>> sqlContext.registerDataFrameAsTable(df, "table1")
        >>> sqlContext.dropTempTable("table1")
        """
        self.sparkSession.catalog.dropTempView(tableName)

    def createExternalTable(self, tableName, path=None, source=None, schema=None, **options):
        """Creates an external table based on the dataset in a data source.

        It returns the DataFrame associated with the external table.

        The data source is specified by the ``source`` and a set of ``options``.
        If ``source`` is not specified, the default data source configured by
        ``spark.sql.sources.default`` will be used.

        Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
        created external table.

        .. versionadded:: 1.3.0

        Returns
        -------
        :class:`DataFrame`
        """
        return self.sparkSession.catalog.createExternalTable(
            tableName, path, source, schema, **options)

    def sql(self, sqlQuery):
        """Returns a :class:`DataFrame` representing the result of the given query.

        .. versionadded:: 1.0.0

        Returns
        -------
        :class:`DataFrame`

        Examples
        --------
        >>> sqlContext.registerDataFrameAsTable(df, "table1")
        >>> df2 = sqlContext.sql("SELECT field1 AS f1, field2 as f2 from table1")
        >>> df2.collect()
        [Row(f1=1, f2='row1'), Row(f1=2, f2='row2'), Row(f1=3, f2='row3')]
        """
        return self.sparkSession.sql(sqlQuery)

    def table(self, tableName):
        """Returns the specified table or view as a :class:`DataFrame`.

        .. versionadded:: 1.0.0

        Returns
        -------
        :class:`DataFrame`

        Examples
        --------
        >>> sqlContext.registerDataFrameAsTable(df, "table1")
        >>> df2 = sqlContext.table("table1")
        >>> sorted(df.collect()) == sorted(df2.collect())
        True
        """
        return self.sparkSession.table(tableName)

    def tables(self, dbName=None):
        """Returns a :class:`DataFrame` containing names of tables in the given database.

        If ``dbName`` is not specified, the current database will be used.

        The returned DataFrame has two columns: ``tableName`` and ``isTemporary``
        (a column with :class:`BooleanType` indicating if a table is a temporary one or not).

        .. versionadded:: 1.3.0

        Parameters
        ----------
        dbName: str, optional
            name of the database to use.

        Returns
        -------
        :class:`DataFrame`

        Examples
        --------
        >>> sqlContext.registerDataFrameAsTable(df, "table1")
        >>> df2 = sqlContext.tables()
        >>> df2.filter("tableName = 'table1'").first()
        Row(namespace='', tableName='table1', isTemporary=True)
        """
        if dbName is None:
            return DataFrame(self._ssql_ctx.tables(), self)
        else:
            return DataFrame(self._ssql_ctx.tables(dbName), self)

    def tableNames(self, dbName=None):
        """Returns a list of names of tables in the database ``dbName``.

        .. versionadded:: 1.3.0

        Parameters
        ----------
        dbName: str
            name of the database to use. Default to the current database.

        Returns
        -------
        list
            list of table names, in string

        >>> sqlContext.registerDataFrameAsTable(df, "table1")
        >>> "table1" in sqlContext.tableNames()
        True
        >>> "table1" in sqlContext.tableNames("default")
        True
        """
        if dbName is None:
            return [name for name in self._ssql_ctx.tableNames()]
        else:
            return [name for name in self._ssql_ctx.tableNames(dbName)]

    @since(1.0)
    def cacheTable(self, tableName):
        """Caches the specified table in-memory."""
        self._ssql_ctx.cacheTable(tableName)

    @since(1.0)
    def uncacheTable(self, tableName):
        """Removes the specified table from the in-memory cache."""
        self._ssql_ctx.uncacheTable(tableName)

    @since(1.3)
    def clearCache(self):
        """Removes all cached tables from the in-memory cache. """
        self._ssql_ctx.clearCache()

    @property
    def read(self):
        """
        Returns a :class:`DataFrameReader` that can be used to read data
        in as a :class:`DataFrame`.

        .. versionadded:: 1.4.0

        Returns
        -------
        :class:`DataFrameReader`
        """
        return DataFrameReader(self)

    @property
    def readStream(self):
        """
        Returns a :class:`DataStreamReader` that can be used to read data streams
        as a streaming :class:`DataFrame`.

        .. versionadded:: 2.0.0

        Notes
        -----
        This API is evolving.

        Returns
        -------
        :class:`DataStreamReader`

        >>> text_sdf = sqlContext.readStream.text(tempfile.mkdtemp())
        >>> text_sdf.isStreaming
        True
        """
        return DataStreamReader(self)

    @property
    def streams(self):
        """Returns a :class:`StreamingQueryManager` that allows managing all the
        :class:`StreamingQuery` StreamingQueries active on `this` context.

        .. versionadded:: 2.0.0

        Notes
        -----
        This API is evolving.
        """
        from pyspark.sql.streaming import StreamingQueryManager
        return StreamingQueryManager(self._ssql_ctx.streams())

class HiveContext(SQLContext):
    """A variant of Spark SQL that integrates with data stored in Hive.

    Configuration for Hive is read from ``hive-site.xml`` on the classpath.
    It supports running both SQL and HiveQL commands.

    .. deprecated:: 2.0.0
        Use SparkSession.builder.enableHiveSupport().getOrCreate().

    Parameters
    ----------
    sparkContext : :class:`SparkContext`
        The SparkContext to wrap.
    jhiveContext : optional
        An optional JVM Scala HiveContext. If set, we do not instantiate a new
        :class:`HiveContext` in the JVM, instead we make all calls to this object.
        This is only for internal use.

    """

    def __init__(self, sparkContext, jhiveContext=None):
        warnings.warn(
            "HiveContext is deprecated in Spark 2.0.0. Please use " +
            "SparkSession.builder.enableHiveSupport().getOrCreate() instead.",
            FutureWarning
        )
        if jhiveContext is None:
            sparkContext._conf.set("spark.sql.catalogImplementation", "hive")
            sparkSession = SparkSession.builder._sparkContext(sparkContext).getOrCreate()
        else:
            sparkSession = SparkSession(sparkContext, jhiveContext.sparkSession())
        SQLContext.__init__(self, sparkContext, sparkSession, jhiveContext)

    @classmethod
    def _createForTesting(cls, sparkContext):
        """(Internal use only) Create a new HiveContext for testing.

        All test code that touches HiveContext *must* go through this method. Otherwise,
        you may end up launching multiple derby instances and encounter with incredibly
        confusing error messages.
        """
        jsc = sparkContext._jsc.sc()
        jtestHive = sparkContext._jvm.org.apache.spark.sql.hive.test.TestHiveContext(jsc, False)
        return cls(sparkContext, jtestHive)

    def refreshTable(self, tableName):
        """Invalidate and refresh all the cached the metadata of the given
        table. For performance reasons, Spark SQL or the external data source
        library it uses might cache certain metadata about a table, such as the
        location of blocks. When those change outside of Spark SQL, users should
        call this function to invalidate the cache.
        """
        self._ssql_ctx.refreshTable(tableName)

def _test():
    import os
    import doctest
    import tempfile
    from pyspark.context import SparkContext
    from pyspark.sql import Row, SQLContext
    import pyspark.sql.context

    os.chdir(os.environ["SPARK_HOME"])

    globs = pyspark.sql.context.__dict__.copy()
    sc = SparkContext('local[4]', 'PythonTest')
    globs['tempfile'] = tempfile
    globs['os'] = os
    globs['sc'] = sc
    globs['sqlContext'] = SQLContext(sc)
    globs['rdd'] = rdd = sc.parallelize(
        [Row(field1=1, field2="row1"),
         Row(field1=2, field2="row2"),
         Row(field1=3, field2="row3")]
    )
    globs['df'] = rdd.toDF()
    jsonStrings = [
        '{"field1": 1, "field2": "row1", "field3":{"field4":11}}',
        '{"field1" : 2, "field3":{"field4":22, "field5": [10, 11]},"field6":[{"field7": "row2"}]}',
        '{"field1" : null, "field2": "row3", "field3":{"field4":33, "field5": []}}'
    ]
    globs['jsonStrings'] = jsonStrings
    globs['json'] = sc.parallelize(jsonStrings)
    (failure_count, test_count) = doctest.testmod(
        pyspark.sql.context, globs=globs,
        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)
    globs['sc'].stop()
    if failure_count:
        sys.exit(-1)

if __name__ == "__main__":
    _test()