"""
========================================
Lasso and Elastic Net for Sparse Signals
========================================

Estimates Lasso and Elastic-Net regression models on a manually generated
sparse signal corrupted with an additive noise. Estimated coefficients are
compared with the ground-truth.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.metrics import r2_score

###############################################################################
# generate some sparse data to play with
np.random.seed(42)

n_samples, n_features = 50, 200
X = np.random.randn(n_samples, n_features)
coef = 3 * np.random.randn(n_features)
inds = np.arange(n_features)
np.random.shuffle(inds)
coef[inds[10:]] = 0  # sparsify coef
y = np.dot(X, coef)

# add noise
y += 0.01 * np.random.normal((n_samples,))

# Split data in train set and test set
n_samples = X.shape[0]
X_train, y_train = X[:n_samples / 2], y[:n_samples / 2]
X_test, y_test = X[n_samples / 2:], y[n_samples / 2:]

###############################################################################
# Lasso
from sklearn.linear_model import Lasso

alpha = 0.1
lasso = Lasso(alpha=alpha)

y_pred_lasso = lasso.fit(X_train, y_train).predict(X_test)
r2_score_lasso = r2_score(y_test, y_pred_lasso)
print(lasso)
print("r^2 on test data : %f" % r2_score_lasso)

###############################################################################
# ElasticNet
from sklearn.linear_model import ElasticNet

enet = ElasticNet(alpha=alpha, l1_ratio=0.7)

y_pred_enet = enet.fit(X_train, y_train).predict(X_test)
r2_score_enet = r2_score(y_test, y_pred_enet)
print(enet)
print("r^2 on test data : %f" % r2_score_enet)

plt.plot(enet.coef_, label='Elastic net coefficients')
plt.plot(lasso.coef_, label='Lasso coefficients')
plt.plot(coef, '--', label='original coefficients')
plt.legend(loc='best')
plt.title("Lasso R^2: %f, Elastic Net R^2: %f"
          % (r2_score_lasso, r2_score_enet))
plt.show()


# sometimes it is nice to have ticklabels centered.  mpl currently
# associates a label with a tick, and the label can be aligned
# 'center', 'left', or 'right' using the horizontal alignment property:
#
#
#   for label in ax.xaxis.get_xticklabels():
#       label.set_horizontalalignment('right')
#
#
# but this doesn't help center the label between ticks.  One solution
# is to "face it".  Use the minor ticks to place a tick centered
# between the major ticks.  Here is an example that labels the months,
# centered between the ticks

import numpy as np
import matplotlib.cbook as cbook
import matplotlib.dates as dates
import matplotlib.ticker as ticker
import matplotlib.pyplot as plt

# load some financial data; apple's stock price
fh = cbook.get_sample_data('aapl.npy.gz')
r = np.load(fh); fh.close()
r = r[-250:]  # get the last 250 days

fig, ax = plt.subplots()
ax.plot(r.date, r.adj_close)

ax.xaxis.set_major_locator(dates.MonthLocator())
ax.xaxis.set_minor_locator(dates.MonthLocator(bymonthday=15))

ax.xaxis.set_major_formatter(ticker.NullFormatter())
ax.xaxis.set_minor_formatter(dates.DateFormatter('%b'))

for tick in ax.xaxis.get_minor_ticks():
    tick.tick1line.set_markersize(0)
    tick.tick2line.set_markersize(0)
    tick.label1.set_horizontalalignment('center')

imid = len(r)/2
ax.set_xlabel(str(r.date[imid].year))
plt.show()


"""
.. _ex-envelope-correlation:

=============================================
Compute envelope correlations in source space
=============================================

Compute envelope correlations of orthogonalized activity [1]_ [2]_ in source
space using resting state CTF data.
"""

# Authors: Eric Larson <larson.eric.d@gmail.com>
#          Sheraz Khan <sheraz@khansheraz.com>
#          Denis Engemann <denis.engemann@gmail.com>
#
# License: BSD (3-clause)

import os.path as op

import numpy as np
import matplotlib.pyplot as plt

import mne
from mne.connectivity import envelope_correlation
from mne.minimum_norm import make_inverse_operator, apply_inverse_epochs
from mne.preprocessing import compute_proj_ecg, compute_proj_eog

data_path = mne.datasets.brainstorm.bst_resting.data_path()
subjects_dir = op.join(data_path, 'subjects')
subject = 'bst_resting'
trans = op.join(data_path, 'MEG', 'bst_resting', 'bst_resting-trans.fif')
src = op.join(subjects_dir, subject, 'bem', subject + '-oct-6-src.fif')
bem = op.join(subjects_dir, subject, 'bem', subject + '-5120-bem-sol.fif')
raw_fname = op.join(data_path, 'MEG', 'bst_resting',
                    'subj002_spontaneous_20111102_01_AUX.ds')

##############################################################################
# Here we do some things in the name of speed, such as crop (which will
# hurt SNR) and downsample. Then we compute SSP projectors and apply them.

raw = mne.io.read_raw_ctf(raw_fname, verbose='error')
raw.crop(0, 60).pick_types(meg=True, eeg=False).load_data().resample(80)
raw.apply_gradient_compensation(3)
projs_ecg, _ = compute_proj_ecg(raw, n_grad=1, n_mag=2)
projs_eog, _ = compute_proj_eog(raw, n_grad=1, n_mag=2, ch_name='MLT31-4407')
raw.info['projs'] += projs_ecg
raw.info['projs'] += projs_eog
raw.apply_proj()
cov = mne.compute_raw_covariance(raw)  # compute before band-pass of interest

##############################################################################
# Now we band-pass filter our data and create epochs.

raw.filter(14, 30)
events = mne.make_fixed_length_events(raw, duration=5.)
epochs = mne.Epochs(raw, events=events, tmin=0, tmax=5.,
                    baseline=None, reject=dict(mag=8e-13), preload=True)
del raw

##############################################################################
# Compute the forward and inverse
# -------------------------------

src = mne.read_source_spaces(src)
fwd = mne.make_forward_solution(epochs.info, trans, src, bem)
inv = make_inverse_operator(epochs.info, fwd, cov)
del fwd, src

##############################################################################
# Compute label time series and do envelope correlation
# -----------------------------------------------------

labels = mne.read_labels_from_annot(subject, 'aparc_sub',
                                    subjects_dir=subjects_dir)
epochs.apply_hilbert()  # faster to apply in sensor space
stcs = apply_inverse_epochs(epochs, inv, lambda2=1. / 9., pick_ori='normal',
                            return_generator=True)
label_ts = mne.extract_label_time_course(
    stcs, labels, inv['src'], return_generator=True)
corr = envelope_correlation(label_ts, verbose=True)

# let's plot this matrix
fig, ax = plt.subplots(figsize=(4, 4))
ax.imshow(corr, cmap='viridis', clim=np.percentile(corr, [5, 95]))
fig.tight_layout()

##############################################################################
# Compute the degree and plot it
# ------------------------------

# sphinx_gallery_thumbnail_number = 2
threshold_prop = 0.15  # percentage of strongest edges to keep in the graph
degree = mne.connectivity.degree(corr, threshold_prop=threshold_prop)
stc = mne.labels_to_stc(labels, degree)
stc = stc.in_label(mne.Label(inv['src'][0]['vertno'], hemi='lh') +
                   mne.Label(inv['src'][1]['vertno'], hemi='rh'))
brain = stc.plot(
    clim=dict(kind='percent', lims=[75, 85, 95]), colormap='gnuplot',
    subjects_dir=subjects_dir, views='dorsal', hemi='both',
    smoothing_steps=25, time_label='Beta band')

##############################################################################
# References
# ----------
# .. [1] Hipp JF, Hawellek DJ, Corbetta M, Siegel M, Engel AK (2012)
#        Large-scale cortical correlation structure of spontaneous
#        oscillatory activity. Nature Neuroscience 15:884–890
# .. [2] Khan S et al. (2018). Maturation trajectories of cortical
#        resting-state networks depend on the mediating frequency band.
#        Neuroimage 174:57–68


"""
================
Confusion matrix
================

Example of confusion matrix usage to evaluate the quality
of the output of a classifier on the iris data set. The
diagonal elements represent the number of points for which
the predicted label is equal to the true label, while
off-diagonal elements are those that are mislabeled by the
classifier. The higher the diagonal values of the confusion
matrix the better, indicating many correct predictions.

The figures show the confusion matrix with and without
normalization by class support size (number of elements
in each class). This kind of normalization can be
interesting in case of class imbalance to have a more
visual interpretation of which class is being misclassified.

Here the results are not as good as they could be as our
choice for the regularization parameter C was not the best.
In real life applications this parameter is usually chosen
using :ref:`grid_search`.

"""

print(__doc__)

import itertools
import numpy as np
import matplotlib.pyplot as plt

from sklearn import svm, datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix

# import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target
class_names = iris.target_names

# Split the data into a training set and a test set
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

# Run classifier, using a model that is too regularized (C too low) to see
# the impact on the results
classifier = svm.SVC(kernel='linear', C=0.01)
y_pred = classifier.fit(X_train, y_train).predict(X_test)

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=class_names,
                      title='Confusion matrix, without normalization')

# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,
                      title='Normalized confusion matrix')

plt.show()


"""
Functions to read your force and event data
"""

__author__ = 'Oliver Lindemann'

import os
import sys
import gzip
from collections import OrderedDict
import numpy as np

TAG_COMMENTS = "#"
TAG_UDPDATA  = TAG_COMMENTS + "UDP"
TAG_DAQEVENTS = TAG_COMMENTS + "T"

def _csv(line):
    return list(map(lambda x: x.strip(), line.split(",")))

def DataFrameDict(data, varnames):
    """data frame: Dict of numpy arrays

    does not require Pandas, but can be easily converted to pandas dataframe
    via pandas.DataFrame(data_frame_dict)

    """

    rtn = OrderedDict()
    for v in varnames:
        rtn[v] = []

    for row in data:
        for v, d in zip(varnames, row):
            rtn[v].append(d)

    return rtn

def data_frame_to_text(data_frame):
    rtn = ",".join(data_frame.keys())
    rtn += "\n"
    for x in np.array(list(data_frame.values())).T:
        rtn += ",".join(x) + "\n"
    return rtn

def read_raw_data(path):
    """reading trigger and udp data

    Returns: data, udp_event, daq_events and comments

            data, udp_event, daq_events: DataFrameDict
            comments: text string
    """

    daq_events = []
    udp_events = []
    comments = ""
    data = []
    varnames = None
    app_dir = os.path.split(sys.argv[0])[0]
    path = os.path.abspath(os.path.join(app_dir, path))

    if path.endswith("gz"):
        fl = gzip.open(path, "rt")
    else:
        fl = open(path, "rt")

    for ln in fl:
        if ln.startswith(TAG_COMMENTS):
            comments += ln
            if ln.startswith(TAG_UDPDATA + ","):
                udp_events.append(_csv(ln[len(TAG_UDPDATA) + 1:]))
            elif ln.startswith(TAG_DAQEVENTS):
                daq_events.append(_csv(ln[len(TAG_DAQEVENTS) + 1:]))
        else:
            # data
            if varnames is None:
                # first row contains varnames
                varnames = _csv(ln)
            else:
                data.append(_csv(ln))
    fl.close()

    return (DataFrameDict(data, varnames),
            DataFrameDict(udp_events, ["time", "value"]),
            DataFrameDict(daq_events, ["time", "value"]),
            comments)


import math
import statistics
from itertools import groupby
from random import randint
from typing import Dict, Tuple, Counter

import pandas as pd

from Src.BioAnalyzer.Analysis.GenePrioritization.Steps.DataIntegration.IntermediateRepresentation.Generators import \
    IntermediateRepresentationGeneratorBase
from Src.BioAnalyzer.Analysis.GenePrioritization.Steps.DataIntegration.IntermediateRepresentation.Transformers.SampleTransformerBase import \
    SampleTransformerBase
from Src.BioDataManagement.CrossCutting.DTOs.ExpressionLevelStatusDto import ExpressionLevelStatusDto

class MicroRnaToGeneTransformer(SampleTransformerBase):
    """
    """
    def __init__(self,
                 intermediateRepresentationGenerator: IntermediateRepresentationGeneratorBase,
                 get_global_diff_values_action,
                 get_mirna_gene_target_action):
        super().__init__(intermediateRepresentationGenerator)
        self.__get_mirna_gene_target_action = get_mirna_gene_target_action
        self.__get_global_diff_values_action = get_global_diff_values_action

    def transform(self, from_sample_matrix: pd.DataFrame, is_highly_significant: bool) -> Tuple[pd.DataFrame, Dict[int, ExpressionLevelStatusDto]]:
        mirna_gene_targets = {mirna.lower(): g for mirna, g in
                              self.__get_mirna_gene_targets(from_sample_matrix.columns.tolist()).items()}

        mirna_samples = self.__get_mirna_samples(from_sample_matrix, mirna_gene_targets)

        id_entrez_list = list(set([id_entrez for mirna_symbol, id_entrez_list in mirna_gene_targets.items()
                                   for id_entrez in id_entrez_list]))

        measure_matrix = dict([(g, []) for g in id_entrez_list])
        key_func = lambda gene: gene[0]

        for patient_id, exp_values in mirna_samples.items():
            gene_values = [(id_entrez,
                            exp_value) for mirna_symbol, exp_value in exp_values.items()
                                       for id_entrez in mirna_gene_targets[mirna_symbol]]

            gene_values = sorted(gene_values, key=key_func)

            for id_entrez, measures in groupby(gene_values, key_func):
                measures = [measure for id_entrez, measure in list(measures) if not math.isnan(measure)]
                measure_matrix[id_entrez].append(float('NaN') if not measures else statistics.mean(measures))

        gene_matrix = pd.DataFrame.from_dict(measure_matrix).dropna(axis=1,how='all')
        gene_matrix = self.intermediateRepresentationGenerator.generate(gene_matrix).dropna(axis=1,how='all')

        return gene_matrix, \
               self.__get_gene_status(mirna_gene_targets, gene_matrix.columns.tolist(), is_highly_significant)

    def __get_mirna_gene_targets(self, mirnas):
        gene_targets = {}
        fe_target = self.__get_mirna_gene_target_action(mirnas)

        gene_targets.update(dict([(t.microrna_symbol, list(set(gene_targets[t.microrna_symbol] + t.id_entrez_genes)))
                                      if t.microrna_symbol in gene_targets
                                      else (t.microrna_symbol, t.id_entrez_genes) for t in fe_target.result_list]))

        return gene_targets

    def __get_mirna_samples(self, from_sample_matrix, mirna_gene_targets):
        from_sample_matrix = from_sample_matrix[list(mirna_gene_targets.keys()) + ['patient_id']]
        from_sample_matrix.set_index("patient_id", drop=True, inplace=True)
        return from_sample_matrix.to_dict(orient="index")

    def __get_gene_status(self, mirna_gene_targets, genes, is_highly_significant):
        diff_mirna = [diff for diff in self.__get_global_diff_values_action(is_highly_significant).result.values
                      if diff.element_id in mirna_gene_targets]

        genes_status = [(g, diff.status) for diff in diff_mirna
                                      for g in mirna_gene_targets[diff.element_id] if g in genes]

        key_func = lambda gene: gene[0]
        genes_status = sorted(genes_status, key=key_func)

        genes_status_dict = {}
        for id_entrez, status in groupby(genes_status, key_func):
            status = list(status)
            status_counter = Counter(status)
            status = [k for k, v in status_counter.most_common()]
            len_status = len(status) - 1

            genes_status_dict[id_entrez] = status[0] if len_status == 1 else status[randint(0, len_status)]

        return dict([(entrez_id, status[1]) for entrez_id, status in genes_status_dict.items()])


"""RAND Health Insurance Experiment Data"""

__docformat__ = 'restructuredtext'

COPYRIGHT   = """This is in the public domain."""
TITLE       = __doc__
SOURCE      = """
The data was collected by the RAND corporation as part of the Health
Insurance Experiment (HIE).

http://www.rand.org/health/projects/hie.html

This data was used in::

    Cameron, A.C. amd Trivedi, P.K. 2005.  `Microeconometrics: Methods
        and Applications,` Cambridge: New York.

And was obtained from: <http://cameron.econ.ucdavis.edu/mmabook/mmadata.html>

See randhie/src for the original data and description.  The data included
here contains only a subset of the original data.  The data varies slightly
compared to that reported in Cameron and Trivedi.
"""

DESCRSHORT  = """The RAND Co. Health Insurance Experiment Data"""

DESCRLONG   = """"""

NOTE        = """::

    Number of observations - 20,190
    Number of variables - 10
    Variable name definitions::

        mdvis   - Number of outpatient visits to an MD
        lncoins - ln(coinsurance + 1), 0 <= coninsurance <= 100
        idp     - 1 if individual deductible plan, 0 otherwise
        lpi     - ln(max(1, annual participation incentive payment))
        fmde    - 0 if idp = 1; ln(max(1, MDE/(0.01 coinsurance))) otherwise
        physlm  - 1 if the person has a physical limitation
        disea   - number of chronic diseases
        hlthg   - 1 if self-rated health is good
        hlthf   - 1 if self-rated health is fair
        hlthp   - 1 if self-rated health is poor
        (Omitted category is excellent self-rated health)
"""

from numpy import recfromtxt, column_stack, array
from statsmodels.datasets import utils as du
from os.path import dirname, abspath

PATH = '%s/%s' % (dirname(abspath(__file__)), 'randhie.csv')

def load():
    """
    Loads the RAND HIE data and returns a Dataset class.

    ----------
    endog - response variable, mdvis
    exog - design

    Returns
    Load instance:
        a class of the data with array attrbutes 'endog' and 'exog'
    """
    data = _get_data()
    return du.process_recarray(data, endog_idx=0, dtype=float)

def load_pandas():
    """
    Loads the RAND HIE data and returns a Dataset class.

    ----------
    endog - response variable, mdvis
    exog - design

    Returns
    Load instance:
        a class of the data with array attrbutes 'endog' and 'exog'
    """
    from pandas import read_csv
    data = read_csv(PATH)
    return du.process_recarray_pandas(data, endog_idx=0)

def _get_data():
    with open(PATH, "rb") as f:
        data = recfromtxt(f, delimiter=",", names=True, dtype=float)
        return data


"""
Main ChemKED module
"""
# Standard libraries
from os.path import exists
from collections import namedtuple
from warnings import warn
from copy import deepcopy
import xml.etree.ElementTree as etree
import xml.dom.minidom as minidom
from itertools import chain

import numpy as np

# Local imports
from .validation import schema, OurValidator, yaml, Q_
from .converters import datagroup_properties, ReSpecTh_to_ChemKED

VolumeHistory = namedtuple('VolumeHistory', ['time', 'volume'])
VolumeHistory.__doc__ = 'Time history of the volume in an RCM experiment. Deprecated, to be removed after PyKED 0.4'  # noqa: E501
VolumeHistory.time.__doc__ = '(`~numpy.ndarray`): the time during the experiment'
VolumeHistory.volume.__doc__ = '(`~numpy.ndarray`): the volume during the experiment'

TimeHistory = namedtuple('TimeHistory', ['time', 'quantity', 'type'])
TimeHistory.__doc__ = 'Time history of the quantity in an RCM experiment'
TimeHistory.time.__doc__ = '(`~numpy.ndarray`): the time during the experiment'
TimeHistory.quantity.__doc__ = '(`~numpy.ndarray`): the quantity of interest during the experiment'
TimeHistory.type.__doc__ = """\
(`str`): the type of time history represented. Possible options are:

* volume
* temperature
* pressure
* piston position
* light emission
* OH emission
* absorption
"""

RCMData = namedtuple(
    'RCMData',
    ['compressed_pressure', 'compressed_temperature', 'compression_time', 'stroke',
     'clearance', 'compression_ratio']
)
RCMData.__doc__ = 'Data fields specific to rapid compression machine experiments'
RCMData.compressed_pressure.__doc__ = '(`~pint.Quantity`) The pressure at the end of compression'
RCMData.compressed_temperature.__doc__ = """\
(`~pint.Quantity`) The temperature at the end of compression"""
RCMData.compression_time.__doc__ = '(`~pint.Quantity`) The duration of the compression stroke'
RCMData.stroke.__doc__ = '(`~pint.Quantity`) The length of the stroke'
RCMData.clearance.__doc__ = """\
(`~pint.Quantity`) The clearance between piston face and end wall at the end of compression"""
RCMData.compression_ratio.__doc__ = '(`~pint.Quantity`) The volumetric compression ratio'

Reference = namedtuple('Reference',
                       ['volume', 'journal', 'doi', 'authors', 'detail', 'year', 'pages'])
Reference.__doc__ = 'Information about the article or report where the data can be found'
Reference.volume.__doc__ = '(`str`) The journal volume'
Reference.journal.__doc__ = '(`str`) The name of the journal'
Reference.doi.__doc__ = '(`str`) The Digital Object Identifier of the article'
Reference.authors.__doc__ = '(`list`) The list of authors of the article'
Reference.detail.__doc__ = '(`str`) Detail about where the data can be found in the article'
Reference.year.__doc__ = '(`str`) The year the article was published'
Reference.pages.__doc__ = '(`str`) The pages in the journal where the article was published'

Apparatus = namedtuple('Apparatus', ['kind', 'institution', 'facility'])
Apparatus.__doc__ = 'Information about the experimental apparatus used to generate the data'
Apparatus.kind.__doc__ = '(`str`) The kind of experimental apparatus'
Apparatus.institution.__doc__ = '(`str`) The institution where the experiment is located'
Apparatus.facility.__doc__ = '(`str`) The particular experimental facility at the location'

Composition = namedtuple('Composition', 'species_name InChI SMILES atomic_composition amount')
Composition.__doc__ = 'Detail of the initial composition of the mixture for the experiment'
Composition.species_name.__doc__ = '(`str`) The name of the species'
Composition.InChI.__doc__ = '(`str`) The InChI identifier for the species'
Composition.SMILES.__doc__ = '(`str`) The SMILES identifier for the species'
Composition.atomic_composition.__doc__ = '(`dict`) The atomic composition of the species'
Composition.amount.__doc__ = '(`~pint.Quantity`) The amount of this species'

class ChemKED(object):
    """Main ChemKED class.

    The ChemKED class stores information about the contents of a ChemKED database
    file. It stores each datapoint associated with the database and provides access
    the the reference information, versions, and file author.

    Arguments:
        yaml_file (`str`, optional): The filename of the YAML database in ChemKED format.
        dict_input (`dict`, optional): A dictionary with the parsed ouput of YAML file in ChemKED
            format.
        skip_validation (`bool`, optional): Whether validation of the ChemKED should be done. Must
            be supplied as a keyword-argument.

    Attributes:
        datapoints (`list`): List of `DataPoint` objects storing each datapoint in the database.
        reference (`~collections.namedtuple`): Attributes include ``volume``, ``journal``, ``doi``,
            ``authors``, ``detail``, ``year``, and ``pages`` describing the reference from which the
            datapoints are derived.
        apparatus (`~collections.namedtuple`): Attributes include ``kind`` of experimental
            apparatus, and the ``institution`` and ``facility`` where the experimental apparatus is
            located.
        chemked_version (`str`): Version of the ChemKED database schema used in this file.
        experiment_type (`str`): Type of exeperimental data contained in this database.
        file_author (`dict`): Information about the author of the ChemKED database file.
        file_version (`str`): Version of the ChemKED database file.
        _properties (`dict`): Original dictionary read from ChemKED database file, meant for
            internal use.
    """
    def __init__(self, yaml_file=None, dict_input=None, *, skip_validation=False):
        if yaml_file is not None:
            with open(yaml_file, 'r') as f:
                self._properties = yaml.safe_load(f)
        elif dict_input is not None:
            self._properties = dict_input
        else:
            raise NameError("ChemKED needs either a YAML filename or dictionary as input.")

        if not skip_validation:
            self.validate_yaml(self._properties)

        self.datapoints = []
        for point in self._properties['datapoints']:
            self.datapoints.append(DataPoint(point))

        self.reference = Reference(
            volume=self._properties['reference'].get('volume'),
            journal=self._properties['reference'].get('journal'),
            doi=self._properties['reference'].get('doi'),
            authors=self._properties['reference'].get('authors'),
            detail=self._properties['reference'].get('detail'),
            year=self._properties['reference'].get('year'),
            pages=self._properties['reference'].get('pages'),
        )

        self.apparatus = Apparatus(
            kind=self._properties['apparatus'].get('kind'),
            institution=self._properties['apparatus'].get('institution'),
            facility=self._properties['apparatus'].get('facility'),
        )

        for prop in ['chemked-version', 'experiment-type', 'file-authors', 'file-version']:
            setattr(self, prop.replace('-', '_'), self._properties[prop])

    @classmethod
    def from_respecth(cls, filename_xml, file_author='', file_author_orcid=''):
        """Construct a ChemKED instance directly from a ReSpecTh file.

        Arguments:
            filename_xml (`str`): Filename of the ReSpecTh-formatted XML file to be imported
            file_author (`str`, optional): File author to be added to the list generated from the
                XML file
            file_author_orcid (`str`, optional): ORCID for the file author being added to the list
                of file authors

        Returns:
            `ChemKED`: Instance of the `ChemKED` class containing the data in ``filename_xml``.

        Examples:
            >>> ck = ChemKED.from_respecth('respecth_file.xml')
            >>> ck = ChemKED.from_respecth('respecth_file.xml', file_author='Bryan W. Weber')
            >>> ck = ChemKED.from_respecth('respecth_file.xml', file_author='Bryan W. Weber',
                                           file_author_orcid='0000-0000-0000-0000')
        """
        properties = ReSpecTh_to_ChemKED(filename_xml, file_author, file_author_orcid,
                                         validate=False)
        return cls(dict_input=properties)

    def validate_yaml(self, properties):
        """Validate the parsed YAML file for adherance to the ChemKED format.

        Arguments:
            properties (`dict`): Dictionary created from the parsed YAML file

        Raises:
            `ValueError`: If the YAML file cannot be validated, a `ValueError` is raised whose
                string contains the errors that are present.
        """
        validator = OurValidator(schema)
        if not validator.validate(properties):
            for key, value in validator.errors.items():
                if any(['unallowed value' in v for v in value]):
                    print(('{key} has an illegal value. Allowed values are {values} and are case '
                           'sensitive.').format(key=key, values=schema[key]['allowed']))

            raise ValueError(validator.errors)

    def get_dataframe(self, output_columns=None):
        """Get a Pandas DataFrame of the datapoints in this instance.

        Arguments:
            output_columns (`list`, optional): List of strings specifying the columns to include
                in the output DataFrame. The default is `None`, which outputs all of the
                columns. Options include (not case sensitive):

                    * ``Temperature``
                    * ``Pressure``
                    * ``Ignition Delay``
                    * ``Composition``
                    * ``Equivalence Ratio``
                    * ``Reference``
                    * ``Apparatus``
                    * ``Experiment Type``
                    * ``File Author``
                    * ``File Version``
                    * ``ChemKED Version``

                In addition, specific fields from the ``Reference`` and ``Apparatus`` attributes can
                be included by specifying the name after a colon. These options are:

                    * ``Reference:Volume``
                    * ``Reference:Journal``
                    * ``Reference:DOI``
                    * ``Reference:Authors``
                    * ``Reference:Detail``
                    * ``Reference:Year``
                    * ``Reference:Pages``
                    * ``Apparatus:Kind``
                    * ``Apparatus:Facility``
                    * ``Apparatus:Institution``

                Only the first author is printed when ``Reference`` or ``Reference:Authors`` is
                selected because the whole author list may be quite long.

        Note:
            If the Composition is selected as an output type, the composition specified in the
            `DataPoint` is used. No attempt is made to convert to a consistent basis; mole fractions
            will remain mole fractions, mass fractions will remain mass fractions, and mole percent
            will remain mole percent. Therefore, it is possible to end up with more than one type of
            composition specification in a given column. However, if the composition is included
            in the resulting dataframe, the type of each composition will be specified by the "Kind"
            field in each row.

        Examples:
            >>> df = ChemKED(yaml_file).get_dataframe()
            >>> df = ChemKED(yaml_file).get_dataframe(['Temperature', 'Ignition Delay'])

        Returns:
            `~pandas.DataFrame`: Contains the information regarding each point in the ``datapoints``
                attribute
        """
        import pandas as pd

        valid_labels = [a.replace('_', ' ') for a in self.__dict__
                        if not (a.startswith('__') or a.startswith('_'))
                        ]
        valid_labels.remove('datapoints')
        valid_labels.extend(
            ['composition', 'ignition delay', 'temperature', 'pressure', 'equivalence ratio']
        )
        ref_index = valid_labels.index('reference')
        valid_labels[ref_index:ref_index + 1] = ['reference:' + a for a in Reference._fields]
        app_index = valid_labels.index('apparatus')
        valid_labels[app_index:app_index + 1] = ['apparatus:' + a for a in Apparatus._fields]
        species_list = list(set(chain(*[list(d.composition.keys()) for d in self.datapoints])))

        if output_columns is None or len(output_columns) == 0:
            col_labels = valid_labels
            comp_index = col_labels.index('composition')
            col_labels[comp_index:comp_index + 1] = species_list + ['Composition:Kind']
        else:
            output_columns = [a.lower() for a in output_columns]
            col_labels = []
            for col in output_columns:
                if col in valid_labels or col in ['reference', 'apparatus']:
                    col_labels.append(col)
                else:
                    raise ValueError('{} is not a valid output column choice'.format(col))

            if 'composition' in col_labels:
                comp_index = col_labels.index('composition')
                col_labels[comp_index:comp_index + 1] = species_list + ['Composition:Kind']
            if 'reference' in col_labels:
                ref_index = col_labels.index('reference')
                col_labels[ref_index:ref_index + 1] = ['reference:' + a for a in Reference._fields]
            if 'apparatus' in col_labels:
                app_index = col_labels.index('apparatus')
                col_labels[app_index:app_index + 1] = ['apparatus:' + a for a in Apparatus._fields]

        data = []
        for d in self.datapoints:
            row = []
            d_species = list(d.composition.keys())
            for col in col_labels:
                if col in species_list:
                    if col in d_species:
                        row.append(d.composition[col].amount)
                    else:
                        row.append(Q_(0.0, 'dimensionless'))
                elif 'reference' in col or 'apparatus' in col:
                    split_col = col.split(':')
                    if split_col[1] == 'authors':
                        row.append(getattr(getattr(self, split_col[0]), split_col[1])[0]['name'])
                    else:
                        row.append(getattr(getattr(self, split_col[0]), split_col[1]))
                elif col in ['temperature', 'pressure', 'ignition delay', 'equivalence ratio']:
                    row.append(getattr(d, col.replace(' ', '_')))
                elif col == 'file authors':
                    row.append(getattr(self, col.replace(' ', '_'))[0]['name'])
                elif col == 'Composition:Kind':
                    row.append(d.composition_type)
                else:
                    row.append(getattr(self, col.replace(' ', '_')))
            data.append(row)

        col_labels = [a.title() for a in col_labels]
        columns = pd.Index(col_labels)
        return pd.DataFrame(data=data, columns=columns)

    def write_file(self, filename, *, overwrite=False):
        """Write new ChemKED YAML file based on object.

        Arguments:
            filename (`str`): Filename for target YAML file
            overwrite (`bool`, optional): Whether to overwrite file with given name if present.
                Must be supplied as a keyword-argument.

        Raises:
            `NameError`: If ``filename`` is already present, and ``overwrite`` is not ``True``.

        Example:
            >>> dataset = ChemKED(yaml_file)
            >>> dataset.write_file(new_yaml_file)
        """
        # Ensure file isn't already present
        if exists(filename) and not overwrite:
            raise OSError(filename + ' already present. Specify "overwrite=True" '
                          'to overwrite, or rename.'
                          )

        with open(filename, 'w') as yaml_file:
            yaml.dump(self._properties, yaml_file)

    def convert_to_ReSpecTh(self, filename):
        """Convert ChemKED record to ReSpecTh XML file.

        This converter uses common information in a ChemKED file to generate a
        ReSpecTh XML file. Note that some information may be lost, as ChemKED stores
        some additional attributes.

        Arguments:
            filename (`str`): Filename for output ReSpecTh XML file.

        Example:
            >>> dataset = ChemKED(yaml_file)
            >>> dataset.convert_to_ReSpecTh(xml_file)
        """
        root = etree.Element('experiment')

        file_author = etree.SubElement(root, 'fileAuthor')
        file_author.text = self.file_authors[0]['name']

        # right now ChemKED just uses an integer file version
        file_version = etree.SubElement(root, 'fileVersion')
        major_version = etree.SubElement(file_version, 'major')
        major_version.text = str(self.file_version)
        minor_version = etree.SubElement(file_version, 'minor')
        minor_version.text = '0'

        respecth_version = etree.SubElement(root, 'ReSpecThVersion')
        major_version = etree.SubElement(respecth_version, 'major')
        major_version.text = '1'
        minor_version = etree.SubElement(respecth_version, 'minor')
        minor_version.text = '0'

        # Only ignition delay currently supported
        exp = etree.SubElement(root, 'experimentType')
        if self.experiment_type == 'ignition delay':
            exp.text = 'Ignition delay measurement'
        else:
            raise NotImplementedError('Only ignition delay type supported for conversion.')

        reference = etree.SubElement(root, 'bibliographyLink')
        citation = ''
        for author in self.reference.authors:
            citation += author['name'] + ', '
        citation += (self.reference.journal + ' (' + str(self.reference.year) + ') ' +
                     str(self.reference.volume) + ':' + self.reference.pages + '. ' +
                     self.reference.detail
                     )
        reference.set('preferredKey', citation)
        reference.set('doi', self.reference.doi)

        apparatus = etree.SubElement(root, 'apparatus')
        kind = etree.SubElement(apparatus, 'kind')
        kind.text = self.apparatus.kind

        common_properties = etree.SubElement(root, 'commonProperties')
        # ChemKED objects have no common properties once loaded. Check for properties
        # among datapoints that tend to be common
        common = []
        composition = self.datapoints[0].composition

        # Composition type *has* to be the same
        composition_type = self.datapoints[0].composition_type
        if not all(dp.composition_type == composition_type for dp in self.datapoints):
            raise NotImplementedError('Error: ReSpecTh does not support varying composition '
                                      'type among datapoints.'
                                      )

        if all([composition == dp.composition for dp in self.datapoints]):
            # initial composition is common
            common.append('composition')
            prop = etree.SubElement(common_properties, 'property')
            prop.set('name', 'initial composition')

            for species_name, species in composition.items():
                component = etree.SubElement(prop, 'component')
                species_link = etree.SubElement(component, 'speciesLink')
                species_link.set('preferredKey', species_name)
                if species.InChI is not None:
                    species_link.set('InChI', species.InChI)

                amount = etree.SubElement(component, 'amount')
                amount.set('units', composition_type)
                amount.text = str(species.amount.magnitude)

        # If multiple datapoints present, then find any common properties. If only
        # one datapoint, then composition should be the only "common" property.
        if len(self.datapoints) > 1:
            for prop_name in datagroup_properties:
                attribute = prop_name.replace(' ', '_')
                quantities = [getattr(dp, attribute, False) for dp in self.datapoints]

                # All quantities must have the property in question and all the
                # values must be equal
                if all(quantities) and quantities.count(quantities[0]) == len(quantities):
                    common.append(prop_name)
                    prop = etree.SubElement(common_properties, 'property')
                    prop.set('description', '')
                    prop.set('name', prop_name)
                    prop.set('units', str(quantities[0].units))

                    value = etree.SubElement(prop, 'value')
                    value.text = str(quantities[0].magnitude)

        # Ignition delay can't be common, unless only a single datapoint.

        datagroup = etree.SubElement(root, 'dataGroup')
        datagroup.set('id', 'dg1')
        datagroup_link = etree.SubElement(datagroup, 'dataGroupLink')
        datagroup_link.set('dataGroupID', '')
        datagroup_link.set('dataPointID', '')

        property_idx = {}
        labels = {'temperature': 'T', 'pressure': 'P',
                  'ignition delay': 'tau', 'pressure rise': 'dP/dt',
                  }

        for prop_name in datagroup_properties:
            attribute = prop_name.replace(' ', '_')
            # This can't be hasattr because properties are set to the value None
            # if no value is specified in the file, so the attribute always exists
            prop_indices = [i for i, dp in enumerate(self.datapoints)
                            if getattr(dp, attribute) is not None
                            ]
            if prop_name in common or not prop_indices:
                continue

            prop = etree.SubElement(datagroup, 'property')
            prop.set('description', '')
            prop.set('name', prop_name)
            units = str(getattr(self.datapoints[prop_indices[0]], attribute).units)
            prop.set('units', units)
            idx = 'x{}'.format(len(property_idx) + 1)
            property_idx[idx] = {'name': prop_name, 'units': units}
            prop.set('id', idx)
            prop.set('label', labels[prop_name])

        # Need to handle datapoints with possibly different species in the initial composition
        if 'composition' not in common:
            for dp in self.datapoints:
                for species in dp.composition.values():
                    # Only add new property for species not already considered
                    has_spec = any([species.species_name in d.values()
                                    for d in property_idx.values()
                                    ])
                    if not has_spec:
                        prop = etree.SubElement(datagroup, 'property')
                        prop.set('description', '')

                        idx = 'x{}'.format(len(property_idx) + 1)
                        property_idx[idx] = {'name': species.species_name}
                        prop.set('id', idx)
                        prop.set('label', '[' + species.species_name + ']')
                        prop.set('name', 'composition')
                        prop.set('units', self.datapoints[0].composition_type)

                        species_link = etree.SubElement(prop, 'speciesLink')
                        species_link.set('preferredKey', species.species_name)
                        if species.InChI is not None:
                            species_link.set('InChI', species.InChI)

        for dp in self.datapoints:
            datapoint = etree.SubElement(datagroup, 'dataPoint')
            for idx, val in property_idx.items():
                # handle regular properties a bit differently than composition
                if val['name'] in datagroup_properties:
                    value = etree.SubElement(datapoint, idx)
                    quantity = getattr(dp, val['name'].replace(' ', '_')).to(val['units'])
                    value.text = str(quantity.magnitude)
                else:
                    # composition
                    for item in dp.composition.values():
                        if item.species_name == val['name']:
                            value = etree.SubElement(datapoint, idx)
                            value.text = str(item.amount.magnitude)

        # See https://stackoverflow.com/a/16097112 for the None.__ne__
        history_types = ['volume_history', 'temperature_history', 'pressure_history',
                         'piston_position_history', 'light_emission_history',
                         'OH_emission_history', 'absorption_history']
        time_histories = [getattr(dp, p) for dp in self.datapoints for p in history_types]
        time_histories = list(filter(None.__ne__, time_histories))

        if len(self.datapoints) > 1 and len(time_histories) > 1:
            raise NotImplementedError('Error: ReSpecTh files do not support multiple datapoints '
                                      'with a time history.')
        elif len(time_histories) > 0:
            for dg_idx, hist in enumerate(time_histories):
                if hist.type not in ['volume', 'temperature', 'pressure']:
                    warn('The time-history type {} is not supported by ReSpecTh for '
                         'ignition delay experiments'.format(hist.type))
                    continue

                datagroup = etree.SubElement(root, 'dataGroup')
                datagroup.set('id', 'dg{}'.format(dg_idx))
                datagroup_link = etree.SubElement(datagroup, 'dataGroupLink')
                datagroup_link.set('dataGroupID', '')
                datagroup_link.set('dataPointID', '')

                # Time history has two properties: time and quantity.
                prop = etree.SubElement(datagroup, 'property')
                prop.set('description', '')
                prop.set('name', 'time')
                prop.set('units', str(hist.time.units))
                time_idx = 'x{}'.format(len(property_idx) + 1)
                property_idx[time_idx] = {'name': 'time'}
                prop.set('id', time_idx)
                prop.set('label', 't')

                prop = etree.SubElement(datagroup, 'property')
                prop.set('description', '')
                prop.set('name', hist.type)
                prop.set('units', str(hist.quantity.units))
                quant_idx = 'x{}'.format(len(property_idx) + 1)
                property_idx[quant_idx] = {'name': hist.type}
                prop.set('id', quant_idx)
                prop.set('label', 'V')

                for time, quantity in zip(hist.time, hist.quantity):
                    datapoint = etree.SubElement(datagroup, 'dataPoint')
                    value = etree.SubElement(datapoint, time_idx)
                    value.text = str(time.magnitude)
                    value = etree.SubElement(datapoint, quant_idx)
                    value.text = str(quantity.magnitude)

        ign_types = [getattr(dp, 'ignition_type', False) for dp in self.datapoints]
        # All datapoints must have the same ignition target and type
        if all(ign_types) and ign_types.count(ign_types[0]) == len(ign_types):
            # In ReSpecTh files all datapoints must share ignition type
            ignition = etree.SubElement(root, 'ignitionType')
            if ign_types[0]['target'] in ['pressure', 'temperature']:
                ignition.set('target', ign_types[0]['target'][0].upper())
            else:
                # options left are species
                ignition.set('target', self.datapoints[0].ignition_type['target'])
            if ign_types[0]['type'] == 'd/dt max extrapolated':
                ignition.set('type', 'baseline max intercept from d/dt')
            else:
                ignition.set('type', self.datapoints[0].ignition_type['type'])
        else:
            raise NotImplementedError('Different ignition targets or types for multiple datapoints '
                                      'are not supported in ReSpecTh.')

        et = etree.ElementTree(root)
        et.write(filename, encoding='utf-8', xml_declaration=True)

        # now do a "pretty" rewrite
        xml = minidom.parse(filename)
        xml_string = xml.toprettyxml(indent='    ')
        with open(filename, 'w') as f:
            f.write(xml_string)

        print('Converted to ' + filename)

class DataPoint(object):
    """Class for a single datapoint.

    The `DataPoint` class stores the information associated with a single data point in the dataset
    parsed from the `ChemKED` YAML input.

    Arguments:
        properties (`dict`): Dictionary adhering to the ChemKED format for ``datapoints``

    Attributes:
        composition (`list`): List of dictionaries representing the species and their quantities
        ignition_delay (pint.Quantity): The ignition delay of the experiment
        temperature (pint.Quantity): The temperature of the experiment
        pressure (pint.Quantity): The pressure of the experiment
        pressure_rise (pint.Quantity, optional): The amount of pressure rise during the induction
            period of a shock tube experiment.
        compression_time (pint.Quantity, optional): The compression time for an RCM experiment.
        compressed_pressure (pint.Quantity, optional): The pressure at the end of compression for
            an RCM experiment.
        compressed_temperature (pint.Quantity, optional): The temperature at the end of compression
            for an RCM experiment.
        first_stage_ignition_delay (pint.Quantity, optional): The first stage ignition delay of the
            experiment.
        compression_time (pint.Quantity, optional): The compression time for an RCM experiment.
        ignition_type (`dict`): Dictionary with the ignition target and type.
        volume_history (`~collections.namedtuple`, optional): The volume history of the reactor
            during an RCM experiment.
        pressure_history (`~collections.namedtuple`, optional): The pressure history of the reactor
            during an experiment.
        temperature_history (`~collections.namedtuple`, optional): The temperature history of the
            reactor during an experiment.
        piston_position_history (`~collections.namedtuple`, optional): The piston position history
            of the reactor during an RCM experiment.
        light_emission_history (`~collections.namedtuple`, optional): The light emission history
            of the reactor during an experiment.
        OH_emission_history (`~collections.namedtuple`, optional): The OH emission history of the
            reactor during an experiment.
        absorption_history (`~collections.namedtuple`, optional): The absorption history of the
            reactor during an experiment.
    """
    value_unit_props = [
        'ignition-delay', 'first-stage-ignition-delay', 'temperature', 'pressure',
        'pressure-rise',
    ]

    rcm_data_props = [
        'compressed-pressure', 'compressed-temperature', 'compression-time', 'stroke', 'clearance',
        'compression-ratio'
    ]

    def __init__(self, properties):
        for prop in self.value_unit_props:
            if prop in properties:
                quant = self.process_quantity(properties[prop])
                setattr(self, prop.replace('-', '_'), quant)
            else:
                setattr(self, prop.replace('-', '_'), None)

        if 'rcm-data' in properties:
            orig_rcm_data = properties['rcm-data']
            rcm_props = {}
            for prop in self.rcm_data_props:
                if prop in orig_rcm_data:
                    quant = self.process_quantity(orig_rcm_data[prop])
                    rcm_props[prop.replace('-', '_')] = quant
                else:
                    rcm_props[prop.replace('-', '_')] = None
            self.rcm_data = RCMData(**rcm_props)
        else:
            self.rcm_data = None

        self.composition_type = properties['composition']['kind']
        composition = {}
        for species in properties['composition']['species']:
            species_name = species['species-name']
            amount = self.process_quantity(species['amount'])
            InChI = species.get('InChI')
            SMILES = species.get('SMILES')
            atomic_composition = species.get('atomic-composition')
            composition[species_name] = Composition(
                species_name=species_name, InChI=InChI, SMILES=SMILES,
                atomic_composition=atomic_composition, amount=amount)

        setattr(self, 'composition', composition)

        self.equivalence_ratio = properties.get('equivalence-ratio')
        self.ignition_type = deepcopy(properties.get('ignition-type'))

        if 'time-histories' in properties and 'volume-history' in properties:
            raise TypeError('time-histories and volume-history are mutually exclusive')

        if 'time-histories' in properties:
            for hist in properties['time-histories']:
                if hasattr(self, '{}_history'.format(hist['type'].replace(' ', '_'))):
                    raise ValueError('Each history type may only be specified once. {} was '
                                     'specified multiple times'.format(hist['type']))
                time_col = hist['time']['column']
                time_units = hist['time']['units']
                quant_col = hist['quantity']['column']
                quant_units = hist['quantity']['units']
                if isinstance(hist['values'], list):
                    values = np.array(hist['values'])
                else:
                    # Load the values from a file
                    values = np.genfromtxt(hist['values']['filename'], delimiter=',')

                time_history = TimeHistory(
                    time=Q_(values[:, time_col], time_units),
                    quantity=Q_(values[:, quant_col], quant_units),
                    type=hist['type'],
                )

                setattr(self, '{}_history'.format(hist['type'].replace(' ', '_')), time_history)

        if 'volume-history' in properties:
            warn('The volume-history field should be replaced by time-histories. '
                 'volume-history will be removed after PyKED 0.4',
                 DeprecationWarning)
            time_col = properties['volume-history']['time']['column']
            time_units = properties['volume-history']['time']['units']
            volume_col = properties['volume-history']['volume']['column']
            volume_units = properties['volume-history']['volume']['units']
            values = np.array(properties['volume-history']['values'])
            self.volume_history = VolumeHistory(
                time=Q_(values[:, time_col], time_units),
                volume=Q_(values[:, volume_col], volume_units),
            )

        history_types = ['volume', 'temperature', 'pressure', 'piston_position', 'light_emission',
                         'OH_emission', 'absorption']
        for h in history_types:
            if not hasattr(self, '{}_history'.format(h)):
                setattr(self, '{}_history'.format(h), None)

    def process_quantity(self, properties):
        """Process the uncertainty information from a given quantity and return it
        """
        quant = Q_(properties[0])
        if len(properties) > 1:
            unc = properties[1]
            uncertainty = unc.get('uncertainty', False)
            upper_uncertainty = unc.get('upper-uncertainty', False)
            lower_uncertainty = unc.get('lower-uncertainty', False)
            uncertainty_type = unc.get('uncertainty-type')
            if uncertainty_type == 'relative':
                if uncertainty:
                    quant = quant.plus_minus(float(uncertainty), relative=True)
                elif upper_uncertainty and lower_uncertainty:
                    warn('Asymmetric uncertainties are not supported. The '
                         'maximum of lower-uncertainty and upper-uncertainty '
                         'has been used as the symmetric uncertainty.')
                    uncertainty = max(float(upper_uncertainty), float(lower_uncertainty))
                    quant = quant.plus_minus(uncertainty, relative=True)
                else:
                    raise ValueError('Either "uncertainty" or "upper-uncertainty" and '
                                     '"lower-uncertainty" need to be specified.')
            elif uncertainty_type == 'absolute':
                if uncertainty:
                    uncertainty = Q_(uncertainty)
                    quant = quant.plus_minus(uncertainty.to(quant.units).magnitude)
                elif upper_uncertainty and lower_uncertainty:
                    warn('Asymmetric uncertainties are not supported. The '
                         'maximum of lower-uncertainty and upper-uncertainty '
                         'has been used as the symmetric uncertainty.')
                    uncertainty = max(Q_(upper_uncertainty), Q_(lower_uncertainty))
                    quant = quant.plus_minus(uncertainty.to(quant.units).magnitude)
                else:
                    raise ValueError('Either "uncertainty" or "upper-uncertainty" and '
                                     '"lower-uncertainty" need to be specified.')
            else:
                raise ValueError('uncertainty-type must be one of "absolute" or "relative"')

        return quant

    def get_cantera_composition_string(self, species_conversion=None):
        """Get the composition in a string format suitable for input to Cantera.

        Returns a formatted string no matter the type of composition. As such, this method
        is not recommended for end users; instead, prefer the `get_cantera_mole_fraction`
        or `get_cantera_mass_fraction` methods.

        Arguments:
            species_conversion (`dict`, optional): Mapping of species identifier to a
                species name. This argument should be supplied when the name of the
                species in the ChemKED YAML file does not match the name of the same
                species in a chemical kinetic mechanism. The species identifier (the key
                of the mapping) can be the name, InChI, or SMILES provided in the ChemKED
                file, while the value associated with a key should be the desired name in
                the Cantera format output string.

        Returns:
            `str`: String in the ``SPEC:AMT, SPEC:AMT`` format

        Raises:
            `ValueError`: If the composition type of the `DataPoint` is not one of
                ``'mass fraction'``, ``'mole fraction'``, or ``'mole percent'``
        """
        if self.composition_type in ['mole fraction', 'mass fraction']:
            factor = 1.0
        elif self.composition_type == 'mole percent':
            factor = 100.0
        else:
            raise ValueError('Unknown composition type: {}'.format(self.composition_type))

        if species_conversion is None:
            comps = ['{!s}:{:.4e}'.format(c.species_name,
                     c.amount.magnitude/factor) for c in self.composition.values()]
        else:
            comps = []
            for c in self.composition.values():
                amount = c.amount.magnitude/factor
                idents = [getattr(c, s, False) for s in ['species_name', 'InChI', 'SMILES']]
                present = [i in species_conversion for i in idents]
                if not any(present):
                    comps.append('{!s}:{:.4e}'.format(c.species_name, amount))
                else:
                    if len([i for i in present if i]) > 1:
                        raise ValueError('More than one conversion present for species {}'.format(
                                         c.species_name))

                    ident = idents[present.index(True)]
                    species_replacement_name = species_conversion.pop(ident)
                    comps.append('{!s}:{:.4e}'.format(species_replacement_name, amount))

            if len(species_conversion) > 0:
                raise ValueError('Unknown species in conversion: {}'.format(species_conversion))

        return ', '.join(comps)

    def get_cantera_mole_fraction(self, species_conversion=None):
        """Get the mole fractions in a string format suitable for input to Cantera.

        Arguments:
            species_conversion (`dict`, optional): Mapping of species identifier to a
                species name. This argument should be supplied when the name of the
                species in the ChemKED YAML file does not match the name of the same
                species in a chemical kinetic mechanism. The species identifier (the key
                of the mapping) can be the name, InChI, or SMILES provided in the ChemKED
                file, while the value associated with a key should be the desired name in
                the Cantera format output string.

        Returns:
            `str`: String of mole fractions in the ``SPEC:AMT, SPEC:AMT`` format

        Raises:
            `ValueError`: If the composition type is ``'mass fraction'``, the conversion cannot
                be done because no molecular weight information is known

        Examples:
            >>> dp = DataPoint(properties)
            >>> dp.get_cantera_mole_fraction()
            'H2:4.4400e-03, O2:5.5600e-03, Ar:9.9000e-01'
            >>> species_conversion = {'H2': 'h2', 'O2': 'o2'}
            >>> dp.get_cantera_mole_fraction(species_conversion)
            'h2:4.4400e-03, o2:5.5600e-03, Ar:9.9000e-01'
            >>> species_conversion = {'1S/H2/h1H': 'h2', '1S/O2/c1-2': 'o2'}
            >>> dp.get_cantera_mole_fraction(species_conversion)
            'h2:4.4400e-03, o2:5.5600e-03, Ar:9.9000e-01'
        """
        if self.composition_type == 'mass fraction':
            raise ValueError('Cannot get mole fractions from the given composition.\n'
                             '{}'.format(self.composition))
        else:
            return self.get_cantera_composition_string(species_conversion)

    def get_cantera_mass_fraction(self, species_conversion=None):
        """Get the mass fractions in a string format suitable for input to Cantera.

        Arguments:
            species_conversion (`dict`, optional): Mapping of species identifier to a
                species name. This argument should be supplied when the name of the
                species in the ChemKED YAML file does not match the name of the same
                species in a chemical kinetic mechanism. The species identifier (the key
                of the mapping) can be the name, InChI, or SMILES provided in the ChemKED
                file, while the value associated with a key should be the desired name in
                the Cantera format output string.

        Returns:
            `str`: String of mass fractions in the ``SPEC:AMT, SPEC:AMT`` format

        Raises:
            `ValueError`: If the composition type is ``'mole fraction'`` or
                ``'mole percent'``, the conversion cannot be done because no molecular
                weight information is known

        Examples:
            >>> dp = DataPoint(properties)
            >>> dp.get_cantera_mass_fraction()
            'H2:2.2525e-04, O2:4.4775e-03, Ar:9.9530e-01'
            >>> species_conversion = {'H2': 'h2', 'O2': 'o2'}
            >>> dp.get_cantera_mass_fraction(species_conversion)
            'h2:2.2525e-04, o2:4.4775e-03, Ar:9.9530e-01'
            >>> species_conversion = {'1S/H2/h1H': 'h2', '1S/O2/c1-2': 'o2'}
            >>> dp.get_cantera_mass_fraction(species_conversion)
            'h2:2.2525e-04, o2:4.4775e-03, Ar:9.9530e-01'
        """
        if self.composition_type in ['mole fraction', 'mole percent']:
            raise ValueError('Cannot get mass fractions from the given composition.\n'
                             '{}'.format(self.composition)
                             )
        else:
            return self.get_cantera_composition_string(species_conversion)


'''
Copyright (C) 2015 Dato, Inc.
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the DATO-PYTHON-LICENSE file for details.
'''
# from nose import with_setup
# -*- coding: utf-8 -*-
from graphlab.data_structures.sarray import SArray
import pandas as pd
import numpy as np
import unittest
import random
import copy
import os
import math
import shutil
import array
import util
import time
import itertools

#######################################################
# Metrics tracking tests are in test_usage_metrics.py #
#######################################################

class SArraySketchTest(unittest.TestCase):
    def setUp(self):
        pass

    def __validate_sketch_result(self, sketch, sa, delta = 1E-7):
        df = pd.DataFrame(list(sa.dropna()))
        pds = pd.Series(list(sa.dropna()))
        if (sa.dtype() == int or sa.dtype() == float):
            if (len(sa) == 0):
                self.assertTrue(math.isnan(sketch.min()))
                self.assertTrue(math.isnan(sketch.min()))
                self.assertEquals(sketch.sum(), 0.0)
                self.assertEquals(sketch.mean(), 0.0)
                self.assertEquals(sketch.var(), 0.0)
                self.assertEquals(sketch.std(), 0.0)
            else:
                self.assertEquals(sketch.min(), sa.min())
                self.assertEquals(sketch.max(), sa.max())
                self.assertEquals(sketch.sum(), sa.sum())
                self.assertAlmostEqual(sketch.mean(), sa.dropna().mean(), delta=delta)
                self.assertAlmostEqual(sketch.var(), sa.dropna().var(), delta=delta)
                self.assertAlmostEqual(sketch.std(), sa.dropna().std(), delta=delta)
                self.assertAlmostEqual(sketch.quantile(0.5), df.quantile(0.5)[0], delta=1)
                self.assertEqual(sketch.quantile(0), df.quantile(0)[0])
                self.assertEqual(sketch.quantile(1), df.quantile(1)[0])

                self.assertEqual(sketch.frequent_items(), SArray(pds).sketch_summary().frequent_items())
                for item in pds.value_counts().index:
                    self.assertEqual(sketch.frequency_count(item), pds.value_counts()[item])

                self.assertAlmostEqual(sketch.num_unique(), len(sa.unique()), delta=3)
        else:
            with self.assertRaises(RuntimeError):
                sketch.quantile((0.5))

        self.assertEqual(sketch.num_undefined(), sa.num_missing())
        self.assertEqual(sketch.size(), len(sa))
        self.assertEqual(sketch.sketch_ready(), True)
        self.assertEqual(sketch.num_elements_processed(), sketch.size())

    def __validate_nested_sketch_result(self, sa):
        sketch = sa.sketch_summary()
        self.__validate_sketch_result(sketch, sa)

        # element length summary
        t = sketch.element_length_summary()
        len_sa = sa.dropna().item_length()
        self.__validate_sketch_result(t, len_sa)

    def test_sketch_int(self):
        int_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, None]
        sa = SArray(data=int_data)
        self.__validate_sketch_result(sa.sketch_summary(), sa)

    def test_sketch_float(self):
        int_data = [1.2, 3,.4, 6.789, None]
        sa = SArray(data=int_data)
        self.__validate_sketch_result(sa.sketch_summary(), sa)

    def test_vector_sketch(self):
        vector_data = [[], [1,2], [3], [4,5,6,7], [8,9,10], None]
        sa = SArray(data=vector_data)

        sketch = sa.sketch_summary();
        self.__validate_sketch_result(sketch, sa)
        self.__validate_sketch_result(sketch.element_length_summary(), sa.dropna().item_length())

        flattened = list(itertools.chain.from_iterable(list(sa.dropna())))
        self.__validate_sketch_result(sketch.element_summary(), SArray(flattened))

        fi = sketch.frequent_items()
        self.assertEqual(len(fi), 5)
        self.assertEqual((fi['[1 2]']), 1)
        self.assertEqual((fi['[4 5 6 7]']), 1)

        # sub sketch with one key
        s = sa.sketch_summary(sub_sketch_keys = 1).element_sub_sketch(1)
        expected = sa.vector_slice(1)
        self.__validate_sketch_result(s, expected)

        # sub sketch with multiple keys
        keys = [1,3]
        s = sa.sketch_summary(sub_sketch_keys = keys).element_sub_sketch(keys)
        self.assertEqual(len(s), len(keys))
        for key in keys:
            self.assertTrue(s.has_key(key))
            expected = sa.vector_slice(key)
            self.__validate_sketch_result(s[key], expected)

        indexes = range(0,10)
        s = sa.sketch_summary(sub_sketch_keys = indexes).element_sub_sketch()
        self.assertEqual(len(s), len(indexes))

    def test_list_sketch(self):
        list_data = [[], [1,2],[1,2], ['a', 'a', 'a', 'b'], [ 1 ,1 , 2], None]
        sa = SArray(list_data)
        self.__validate_nested_sketch_result(sa)
        sketch = sa.sketch_summary();

        self.assertEqual(sketch.num_unique(), 4)
        element_summary = sketch.element_summary()
        another_rep = list(itertools.chain.from_iterable(list(sa.dropna())))
        self.__validate_sketch_result(element_summary, SArray(another_rep, str))

        fi = sketch.frequent_items()
        self.assertEqual(len(fi), 4)
        self.assertEqual((fi['[1,2]']), 2)
        self.assertEqual((fi['["a","a","a","b"]']), 1)

    def test_dict_sketch_int_value(self):
        dict_data = [{}, {'a':1, 'b':2}, {'a':1, 'b':2}, {'a':3, 'c':1}, {'a': 1, 'b': 2, 'c': 3}, None]
        sa = SArray(data=dict_data)
        self.__validate_nested_sketch_result(sa)

        sketch = sa.sketch_summary()
        self.assertEqual(sketch.num_unique(), 4)
        fi = sketch.frequent_items()
        self.assertEqual(len(fi), 4)
        self.assertEqual((fi['{"a":1, "b":2}']), 2)
        self.assertEqual((fi['{"a":3, "c":1}']), 1)

        # Get dict key sketch
        key_summary = sketch.dict_key_summary()
        another_rep = list(itertools.chain.from_iterable(list(sa.dict_keys().dropna())))
        self.__validate_sketch_result(key_summary, SArray(another_rep))

        # Get dict value sketch
        value_summary = sketch.dict_value_summary()
        another_rep = list(itertools.chain.from_iterable(list(sa.dict_values().dropna())))
        self.__validate_sketch_result(value_summary, SArray(another_rep))

        # sub sketch with one key
        s = sa.sketch_summary(sub_sketch_keys ='a').element_sub_sketch('a')
        expected = sa.unpack(column_name_prefix="")['a']
        self.__validate_sketch_result(s, expected)

        s = sa.sketch_summary(sub_sketch_keys ='Nonexist').element_sub_sketch('Nonexist')
        self.assertEqual(s.num_undefined(), len(sa))

        # sub sketch with multiple keys
        keys = ['a', 'b']
        s = sa.sketch_summary(sub_sketch_keys =keys).element_sub_sketch(keys)
        self.assertEqual(len(s), len(keys))
        for key in keys:
            self.assertTrue(s.has_key(key))
            expected = sa.unpack(column_name_prefix="")[key]
            self.__validate_sketch_result(s[key], expected)

    def test_dict_sketch_str_value(self):
        # Dict value sketch type should be auto inferred
        dict_data = [{'a':'b', 'b':'c'}, {'a':'b', 'b':'c'}, {'a':'d', 'b':'4'}, None]
        sa = SArray(data=dict_data)
        self.__validate_nested_sketch_result(sa)

        sketch = sa.sketch_summary()
        fi = sketch.frequent_items()
        self.assertEqual(len(fi), 2)
        self.assertEqual(fi['{"a":"b", "b":"c"}'], 2)
        self.assertEqual(fi['{"a":"d", "b":"4"}'], 1)

        # Get dict key sketch
        key_summary = sketch.dict_key_summary()
        another_rep = list(itertools.chain.from_iterable(list(sa.dict_keys().dropna())))
        self.__validate_sketch_result(key_summary, SArray(another_rep))

        # Get dict value sketch
        value_summary = sketch.dict_value_summary()
        another_rep = list(itertools.chain.from_iterable(list(sa.dict_values().dropna())))
        self.__validate_sketch_result(value_summary, SArray(another_rep))

        # sub sketch with one key
        s = sa.sketch_summary(sub_sketch_keys ='a').element_sub_sketch('a')
        expected = sa.unpack(column_name_prefix="")['a']
        self.__validate_sketch_result(s, expected)

        s = sa.sketch_summary(sub_sketch_keys ='Nonexist').element_sub_sketch('Nonexist')
        self.assertEqual(s.num_undefined(), len(sa))

        # sub sketch with multiple keys
        keys = ['a', 'b']
        s = sa.sketch_summary(sub_sketch_keys =keys).element_sub_sketch(keys)
        self.assertEqual(len(s), len(keys))
        for key in keys:
            self.assertTrue(s.has_key(key))
            expected = sa.unpack(column_name_prefix="")[key]
            self.__validate_sketch_result(s[key], expected)

        # allow pass in empty keys, which will retrieve all keys
        s = sa.sketch_summary(sub_sketch_keys =keys).element_sub_sketch()
        self.assertEqual(len(s), len(keys))
        for key in keys:
            self.assertTrue(s.has_key(key))
            expected = sa.unpack(column_name_prefix="")[key]
            self.__validate_sketch_result(s[key], expected)

    def test_str_sketch(self):
        str_data = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", None]
        sa = SArray(data=str_data)
        sketch = sa.sketch_summary()
        with self.assertRaises(RuntimeError):
          sketch.min()
        with self.assertRaises(RuntimeError):
          sketch.max()
        with self.assertRaises(RuntimeError):
          sketch.sum()
        with self.assertRaises(RuntimeError):
          sketch.mean()
        with self.assertRaises(RuntimeError):
          sketch.var()
        with self.assertRaises(RuntimeError):
          sketch.std()

        self.assertAlmostEqual(sketch.num_unique(), 10, delta=3)
        self.assertEqual(sketch.num_undefined(), 1)
        self.assertEqual(sketch.size(), len(str_data))

        with self.assertRaises(RuntimeError):
          sketch.quantile(0.5)
        self.assertEqual(sketch.frequency_count("1"), 1)
        self.assertEqual(sketch.frequency_count("2"), 1)
        t = sketch.frequent_items()
        self.assertEqual(len(t), 10)

    def test_empty_sketch(self):
        int_data = []
        sa = SArray(data=int_data)
        sketch = sa.sketch_summary()
        self.assertTrue(math.isnan(sketch.min()))
        self.assertTrue(math.isnan(sketch.max()))
        self.assertEquals(sketch.sum(), 0)
        self.assertEqual(sketch.mean(), 0)
        self.assertEqual(sketch.var(), 0)
        self.assertEqual(sketch.std(), 0)
        self.assertEqual(sketch.num_unique(), 0)
        self.assertEqual(sketch.num_undefined(),0)
        self.assertEqual(sketch.size(), 0)
        with self.assertRaises(RuntimeError):
          sketch.quantile(0.5)

        t = sketch.frequent_items()
        self.assertEqual(len(t), 0)

    def test_background_sketch(self):
        dict_data = [{str(i):1} for i in range(1,10000)]
        sa = SArray(dict_data)
        s = sa.sketch_summary(background=True, sub_sketch_keys=[str(i ) for i in range(100,200)])
        s.sketch_ready() # cannot check the actual value as it depends on the speed of processing
        t = s.element_sub_sketch([str(i) for i in range(100, 105)])
        self.assertEqual(len(t), 5)

    def test_large_value_sketch(self):
        sa = SArray([1234567890 for i in range(100)])
        sk = sa.sketch_summary();
        self.__validate_sketch_result(sa.sketch_summary(), sa, 1E-5)

    def test_cancelation(self):
        sa = SArray(range(1,10000))
        s = sa.sketch_summary(background=True)
        s.cancel()
        # this can be rather non-deterministic, so there is very little
        # real output validation that can be done...


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Pipelining: chaining a PCA and a logistic regression
=========================================================

The PCA does an unsupervised dimensionality reduction, while the logistic
regression does the prediction.

We use a GridSearchCV to set the dimensionality of the PCA

"""
print(__doc__)

# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model, decomposition, datasets
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV

logistic = linear_model.LogisticRegression()

pca = decomposition.PCA()
pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])

digits = datasets.load_digits()
X_digits = digits.data
y_digits = digits.target

# Plot the PCA spectrum
pca.fit(X_digits)

plt.figure(1, figsize=(4, 3))
plt.clf()
plt.axes([.2, .2, .7, .7])
plt.plot(pca.explained_variance_, linewidth=2)
plt.axis('tight')
plt.xlabel('n_components')
plt.ylabel('explained_variance_')

# Prediction
n_components = [20, 40, 64]
Cs = np.logspace(-4, 4, 3)

# Parameters of pipelines can be set using ‘__’ separated parameter names:
estimator = GridSearchCV(pipe,
                         dict(pca__n_components=n_components,
                              logistic__C=Cs))
estimator.fit(X_digits, y_digits)

plt.axvline(estimator.best_estimator_.named_steps['pca'].n_components,
            linestyle=':', label='n_components chosen')
plt.legend(prop=dict(size=12))
plt.show()