"""
======================================================
Classification of text documents using sparse features
======================================================

This is an example showing how scikit-learn can be used to classify documents
by topics using a bag-of-words approach. This example uses a scipy.sparse
matrix to store the features and demonstrates various classifiers that can
efficiently handle sparse matrices.

The dataset used in this example is the 20 newsgroups dataset. It will be
automatically downloaded, then cached.

The bar plot indicates the accuracy, training time (normalized) and test time
(normalized) of each classifier.

"""

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Lars Buitinck <L.J.Buitinck@uva.nl>
# License: BSD 3 clause

from __future__ import print_function

import logging
import numpy as np
from optparse import OptionParser
import sys
from time import time
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import RidgeClassifier
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.ensemble import RandomForestClassifier
from sklearn.utils.extmath import density
from sklearn import metrics

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

# parse commandline arguments
op = OptionParser()
op.add_option("--report",
              action="store_true", dest="print_report",
              help="Print a detailed classification report.")
op.add_option("--chi2_select",
              action="store", type="int", dest="select_chi2",
              help="Select some number of features using a chi-squared test")
op.add_option("--confusion_matrix",
              action="store_true", dest="print_cm",
              help="Print the confusion matrix.")
op.add_option("--top10",
              action="store_true", dest="print_top10",
              help="Print ten most discriminative terms per class"
                   " for every classifier.")
op.add_option("--all_categories",
              action="store_true", dest="all_categories",
              help="Whether to use all categories or not.")
op.add_option("--use_hashing",
              action="store_true",
              help="Use a hashing vectorizer.")
op.add_option("--n_features",
              action="store", type=int, default=2 ** 16,
              help="n_features when using the hashing vectorizer.")
op.add_option("--filtered",
              action="store_true",
              help="Remove newsgroup information that is easily overfit: "
                   "headers, signatures, and quoting.")

(opts, args) = op.parse_args()
if len(args) > 0:
    op.error("this script takes no arguments.")
    sys.exit(1)

print(__doc__)
op.print_help()
print()

###############################################################################
# Load some categories from the training set
if opts.all_categories:
    categories = None
else:
    categories = [
        'alt.atheism',
        'talk.religion.misc',
        'comp.graphics',
        'sci.space',
    ]

if opts.filtered:
    remove = ('headers', 'footers', 'quotes')
else:
    remove = ()

print("Loading 20 newsgroups dataset for categories:")
print(categories if categories else "all")

data_train = fetch_20newsgroups(subset='train', categories=categories,
                                shuffle=True, random_state=42,
                                remove=remove)

data_test = fetch_20newsgroups(subset='test', categories=categories,
                               shuffle=True, random_state=42,
                               remove=remove)
print('data loaded')

categories = data_train.target_names    # for case categories == None

def size_mb(docs):
    return sum(len(s.encode('utf-8')) for s in docs) / 1e6

data_train_size_mb = size_mb(data_train.data)
data_test_size_mb = size_mb(data_test.data)

print("%d documents - %0.3fMB (training set)" % (
    len(data_train.data), data_train_size_mb))
print("%d documents - %0.3fMB (test set)" % (
    len(data_test.data), data_test_size_mb))
print("%d categories" % len(categories))
print()

# split a training set and a test set
y_train, y_test = data_train.target, data_test.target

print("Extracting features from the training data using a sparse vectorizer")
t0 = time()
if opts.use_hashing:
    vectorizer = HashingVectorizer(stop_words='english', non_negative=True,
                                   n_features=opts.n_features)
    X_train = vectorizer.transform(data_train.data)
else:
    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
                                 stop_words='english')
    X_train = vectorizer.fit_transform(data_train.data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_train_size_mb / duration))
print("n_samples: %d, n_features: %d" % X_train.shape)
print()

print("Extracting features from the test data using the same vectorizer")
t0 = time()
X_test = vectorizer.transform(data_test.data)
duration = time() - t0
print("done in %fs at %0.3fMB/s" % (duration, data_test_size_mb / duration))
print("n_samples: %d, n_features: %d" % X_test.shape)
print()

# mapping from integer feature name to original token string
if opts.use_hashing:
    feature_names = None
else:
    feature_names = vectorizer.get_feature_names()

if opts.select_chi2:
    print("Extracting %d best features by a chi-squared test" %
          opts.select_chi2)
    t0 = time()
    ch2 = SelectKBest(chi2, k=opts.select_chi2)
    X_train = ch2.fit_transform(X_train, y_train)
    X_test = ch2.transform(X_test)
    if feature_names:
        # keep selected feature names
        feature_names = [feature_names[i] for i
                         in ch2.get_support(indices=True)]
    print("done in %fs" % (time() - t0))
    print()

if feature_names:
    feature_names = np.asarray(feature_names)

def trim(s):
    """Trim string to fit on terminal (assuming 80-column display)"""
    return s if len(s) <= 80 else s[:77] + "..."

###############################################################################
# Benchmark classifiers
def benchmark(clf):
    print('_' * 80)
    print("Training: ")
    print(clf)
    t0 = time()
    clf.fit(X_train, y_train)
    train_time = time() - t0
    print("train time: %0.3fs" % train_time)

    t0 = time()
    pred = clf.predict(X_test)
    test_time = time() - t0
    print("test time:  %0.3fs" % test_time)

    score = metrics.accuracy_score(y_test, pred)
    print("accuracy:   %0.3f" % score)

    if hasattr(clf, 'coef_'):
        print("dimensionality: %d" % clf.coef_.shape[1])
        print("density: %f" % density(clf.coef_))

        if opts.print_top10 and feature_names is not None:
            print("top 10 keywords per class:")
            for i, category in enumerate(categories):
                top10 = np.argsort(clf.coef_[i])[-10:]
                print(trim("%s: %s"
                      % (category, " ".join(feature_names[top10]))))
        print()

    if opts.print_report:
        print("classification report:")
        print(metrics.classification_report(y_test, pred,
                                            target_names=categories))

    if opts.print_cm:
        print("confusion matrix:")
        print(metrics.confusion_matrix(y_test, pred))

    print()
    clf_descr = str(clf).split('(')[0]
    return clf_descr, score, train_time, test_time

results = []
for clf, name in (
        (RidgeClassifier(tol=1e-2, solver="lsqr"), "Ridge Classifier"),
        (Perceptron(n_iter=50), "Perceptron"),
        (PassiveAggressiveClassifier(n_iter=50), "Passive-Aggressive"),
        (KNeighborsClassifier(n_neighbors=10), "kNN"),
        (RandomForestClassifier(n_estimators=100), "Random forest")):
    print('=' * 80)
    print(name)
    results.append(benchmark(clf))

for penalty in ["l2", "l1"]:
    print('=' * 80)
    print("%s penalty" % penalty.upper())
    # Train Liblinear model
    results.append(benchmark(LinearSVC(loss='l2', penalty=penalty,
                                            dual=False, tol=1e-3)))

    # Train SGD model
    results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                           penalty=penalty)))

# Train SGD with Elastic Net penalty
print('=' * 80)
print("Elastic-Net penalty")
results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                       penalty="elasticnet")))

# Train NearestCentroid without threshold
print('=' * 80)
print("NearestCentroid (aka Rocchio classifier)")
results.append(benchmark(NearestCentroid()))

# Train sparse Naive Bayes classifiers
print('=' * 80)
print("Naive Bayes")
results.append(benchmark(MultinomialNB(alpha=.01)))
results.append(benchmark(BernoulliNB(alpha=.01)))

print('=' * 80)
print("LinearSVC with L1-based feature selection")
# The smaller C, the stronger the regularization.
# The more regularization, the more sparsity.
results.append(benchmark(Pipeline([
  ('feature_selection', LinearSVC(penalty="l1", dual=False, tol=1e-3)),
  ('classification', LinearSVC())
])))

# make some plots

indices = np.arange(len(results))

results = [[x[i] for x in results] for i in range(4)]

clf_names, score, training_time, test_time = results
training_time = np.array(training_time) / np.max(training_time)
test_time = np.array(test_time) / np.max(test_time)

plt.figure(figsize=(12, 8))
plt.title("Score")
plt.barh(indices, score, .2, label="score", color='navy')
plt.barh(indices + .3, training_time, .2, label="training time",
         color='c')
plt.barh(indices + .6, test_time, .2, label="test time", color='darkorange')
plt.yticks(())
plt.legend(loc='best')
plt.subplots_adjust(left=.25)
plt.subplots_adjust(top=.95)
plt.subplots_adjust(bottom=.05)

for i, c in zip(indices, clf_names):
    plt.text(-.3, i, c)

plt.show()


# -*- python -*-
# -*- coding: utf-8 -*-
#
#  This file is part of the colormap software
#
#  Copyright (c) 2011-20134
#
#  File author(s): Thomas Cokelaer <cokelaer@gmail.com>
#
#  Distributed under the GPLv3 License.
#  See accompanying file LICENSE.txt or copy at
#      http://www.gnu.org/licenses/gpl-3.0.html
#
#  Website: https://github.com/cokelaer/colormap
#  Documentation: http://packages.python.org/colormap
#
##############################################################################
"""Utilities provided in this module can be found either in the
standard Python module called :mod:`colorsys` or in matplotlib.colors
(e.g rgb2hex) or are original to this module (e.g., rgb2huv)

"""
# matplotlib dependence is only inside Colormap class
import colorsys
from easydev.tools import check_param_in_list, swapdict, check_range
from colormap.xfree86 import XFree86_colors

__all__ = ["HEX", "Color", "hex2web", "web2hex", "hex2rgb", "hex2dec",
    "rgb2hex", "rgb2hsv", "hsv2rgb", "rgb2hls", "hls2rgb","yuv2rgb", "rgb2yuv",
    "to_intensity", "yuv2rgb_int", "rgb2yuv_int", "Colormap"
    ]

def hex2web(hexa):
    """Convert hexadecimal string (6 digits) into *web* version (3 digits)

    .. doctest::

        >>> from colormap.colors import hex2web
        >>> hex2web("#FFAA11")
        '#FA1'

    .. seealso:: :func:`web2hex`, :func:`hex2rgb`
        :func:`rgb2hex`, :func:`rgb2hsv`, :func:`hsv2rgb`, :func:`rgb2hls`,
        :func:`hls2rgb`
    """
    hexa = HEX().get_standard_hex_color(hexa)
    return "#" + hexa[1::2]

def web2hex(web):
    """Convert *web* hexadecimal string (3 digits) into  6 digits version

    .. doctest::

        >>> from colormap.colors import web2hex
        >>> web2hex("#FA1")
        '#FFAA11'

    .. seealso:: :func:`hex2web`, :func:`hex2rgb`
        :func:`rgb2hex`, :func:`rgb2hsv`, :func:`hsv2rgb`, :func:`rgb2hls`,
        :func:`hls2rgb`
    """
    return HEX().get_standard_hex_color(web)

def hex2rgb(hexcolor, normalise=False):
    """This function converts a hex color triplet into RGB

    Valid hex code are:

     * #FFF
     * #0000FF
     * 0x0000FF
     * 0xFA1

    .. doctest::

        >>> from colormap.colors import hex2rgb
        >>> hex2rgb("#FFF", normalise=False)
        (255, 255, 255)
        >>> hex2rgb("#FFFFFF", normalise=True)
        (1.0, 1.0, 1.0)

    .. seealso:: :func:`hex2web`, :func:`web2hex`,
        :func:`rgb2hex`, :func:`rgb2hsv`, :func:`hsv2rgb`, :func:`rgb2hls`,
        :func:`hls2rgb`
    """
    hexcolor = HEX().get_standard_hex_color(hexcolor)[1:]
    r, g, b =  int(hexcolor[0:2], 16), int(hexcolor[2:4], 16), int(hexcolor[4:6], 16)
    if normalise:
        r, g, b = _normalise(r, g, b)
    return r, g, b

def rgb2hex(r, g, b, normalised=False):
    """Convert RGB to hexadecimal color

    :param: can be a tuple/list/set of 3 values (R,G,B)
    :return: a hex vesion ofthe RGB 3-tuple

    .. doctest::

        >>> from colormap.colors import rgb2hex
        >>> rgb2hex(0,0,255, normalised=False)
        '#0000FF'
        >>> rgb2hex(0,0,1, normalised=True)
        '#0000FF'

    .. seealso:: :func:`hex2web`, :func:`web2hex`, :func:`hex2rgb`
        , :func:`rgb2hsv`, :func:`hsv2rgb`, :func:`rgb2hls`,
        :func:`hls2rgb`

    """
    if normalised:
        r, g, b = _denormalise(r, g, b, mode="rgb")
        r = int(r)
        g = int(g)
        b = int(b)
    check_range(r, 0, 255)
    check_range(g, 0, 255)
    check_range(b, 0, 255)
    return '#%02X%02X%02X' % (r, g, b)

def rgb2hls(r, g, b, normalised=True):
    """Convert an RGB value to an HLS value.

    :param bool normalised: if *normalised* is True, the input RGB triplet
        should be in the range 0-1 (0-255 otherwise)
    :return: the HLS triplet. If *normalised* parameter is True, the output
        triplet is in the range 0-1; otherwise, H in the range 0-360 and LS
        in the range 0-100.

    .. doctest::

        >>> from colormap.colors import rgb2hls
        >>> rgb2hls(255,255,255, normalised=False)
        (0.0, 1.0, 0.0)

    .. seealso:: :func:`hex2web`, :func:`web2hex`, :func:`hex2rgb`
        :func:`rgb2hex`, :func:`hsv2rgb`,
        :func:`hls2rgb`
    """
    # rgb_to_hsv expects normalised values !
    if normalised:
        upper = 1
    else:
        upper = 255
    check_range(r, 0, upper)
    check_range(g, 0, upper)
    check_range(b, 0, upper)
    if normalised==False:
        r, g, b = _normalise(r, g, b)
    h, l, s = colorsys.rgb_to_hls(r, g, b)
    return h, l, s

def rgb2hsv(r, g, b, normalised=True):
    """Convert an RGB value to an HSV value.

    :param bool normalised: if *normalised* is True, the input RGB triplet
        should be in the range 0-1 (0-255 otherwise)
    :return: the HSV triplet. If *normalised* parameter is True, the output
        triplet is in the range 0-1; otherwise, H in the range 0-360 and LS
        in the range 0-100.

    .. doctest::

        >>> from colormap.colors import rgb2hsv
        >>> rgb2hsv(0.5,0,1)
        (0.75, 1, 1)

    .. seealso:: :func:`hex2web`, :func:`web2hex`, :func:`hex2rgb`
        :func:`rgb2hex`, :func:`hsv2rgb`, :func:`rgb2hls`,
        :func:`hls2rgb`
    """
    # rgb_to_hsv expects normalised values !
    if normalised:
        upper = 1
    else:
        upper = 255
    check_range(r, 0, upper)
    check_range(g, 0, upper)
    check_range(b, 0, upper)
    if normalised==False:
        r, g, b = _normalise(r, g, b)
    h, s, v = colorsys.rgb_to_hsv(r, g, b)
    return h,s,v

def hsv2rgb(h, s, v, normalised=True):
    """Convert a hue-saturation-value (HSV) value to a red-green-blue (RGB).

    :param bool normalised: If *normalised* is True, the input HSV triplet
        should be in the range 0-1; otherwise, H in the range 0-360 and LS
        in the range 0-100.
    :return: the RGB triplet. The output
        triplet is in the range 0-1 whether the input is normalised or not.

    .. doctest::

        >>> from colormap.colors import hsv2rgb
        >>> hsv2rgb(0.5,1,1, normalised=True)  # doctest: +SKIP
        (0, 1, 1)

    .. seealso:: :func:`hex2web`, :func:`web2hex`, :func:`hex2rgb`
        :func:`rgb2hex`, :func:`rgb2hsv`, :func:`rgb2hls`,
        :func:`hls2rgb`
    .. seealso:: :func:`rgb2hex`
    """
    if normalised:
        upper = 1
    else:
        upper = 100
    if normalised:
        uppera = 1
    else:
        uppera = 360
    check_range(h, 0, uppera)
    check_range(s, 0, upper)
    check_range(v, 0, upper)
    if normalised == False:
        h, s, v = _normalise(h, s, v, mode="hsv")
    return colorsys.hsv_to_rgb(h, s, v)

def hls2rgb(h, l, s, normalised=True):
    """Convert an HLS value to a RGB value.

    :param bool normalised: If *normalised* is True, the input HLS triplet
        should be in the range 0-1; otherwise, H in the range 0-360 and LS
        in the range 0-100.

    :return: the RGB triplet. The output
        triplet is in the range 0-1 whether the input is normalised or not.

    .. doctest::

        >>> from colormap.colors import hls2rgb
        >>> hls2rgb(360, 50, 60, normalised=False)  # doctest: +SKIP
        (0.8, 0.2, 0.2)

    .. seealso:: :func:`hex2web`, :func:`web2hex`, :func:`hex2rgb`
        :func:`rgb2hex`, :func:`rgb2hsv`, :func:`hsv2rgb`, :func:`rgb2hls`,

    """
    if normalised:
        upper = 1
    else:
        upper = 100
    if normalised:
        uppera = 1
    else:
        uppera = 360
    check_range(h, 0, uppera)
    check_range(s, 0, upper)
    check_range(l, 0, upper)
    if normalised == False:
        h, l, s = _normalise(h, l, s, mode="hls")
    return colorsys.hls_to_rgb(h, l, s)

def hex2dec(data):
    """convert hexadecimal string (data) into a float in the [0-65536] inclusive range"""
    if data[0] == '#':
        data.replace('#', '')
    return int(data, 16)/255.

def rgb2yuv(r, g, b):
    """Convert RGB triplet into YUV

    :return: YUV triplet with values between 0 and 1

    `YUV wikipedia <http://en.wikipedia.org/wiki/YUV>`_

    .. warning:: expected input must be between 0 and 1
    .. note:: the constants referenc used is  Rec. 601
    """
    check_range(r, 0, 1)
    check_range(g, 0, 1)
    check_range(b, 0, 1)

    #y = int(0.299 * r + 0.587 * g + 0.114 * b)
    #u = int(-0.14713 * r + -0.28886 * g + 0.436 * b)
    #v = int(0.615 * r + -0.51499 * g + -0.10001 * b)

    y = 0.299 * r + 0.587 * g + 0.114 * b
    u = -32591.0/221500.0 * r + -63983.0/221500.0 * g + 0.436 * b
    v = 0.615 * r + -72201./140200 * g + -7011/70100. * b
    return (y, u, v)

def yuv2rgb(y, u, v):
    """Convert YUV triplet into RGB

    `YUV <http://en.wikipedia.org/wiki/YUV>`_

    .. warning:: expected input must be between 0 and 255 (not normalised)

    """
    check_range(y, 0,1)
    check_range(u, 0, 1)
    check_range(v, 0, 1)
    A, B, C, D = 701.0/615.0, 25251.0/63983.0, 209599.0/361005.0, 443.0/218.0
    r = y + A * v
    g = y - B * u - C * v
    b = y + D * u
    return (r, g, b)

def rgb2yuv_int(r, g, b):
    """Convert RGB triplet into YUV

    `YUV wikipedia <http://en.wikipedia.org/wiki/YUV>`_

    .. warning:: expected input must be between 0 and 255 (not normalised)

    """
    check_range(r, 0, 255)
    check_range(g, 0, 255)
    check_range(b, 0, 255)

    y = int(0.299 * r + 0.587 * g + 0.114 * b)
    u = int(-32591.0/221500.0 * r + -63983.0/221500.0 * g + 0.436 * b)
    v = int(0.615 * r + -72201./140200 * g + -7011/70100. * b)

    return (y, u, v)

def yuv2rgb_int(y, u, v):
    """Convert YUV triplet into RGB

    `YUV <http://en.wikipedia.org/wiki/YUV>`_

    .. warning:: expected input must be between 0 and 255 (not normalised)

    """
    check_range(y, 0, 255)
    check_range(u, 0, 255)
    check_range(v, 0, 255)
    r = int(y + 1.13983 * v)
    g = int(y - 0.39465 * u - 0.58060 * v)
    b = int(y + 2.03211 * u)
    return (r, g, b)

def _denormalise(r, g, b, mode="rgb"):
    check_param_in_list(mode, ["rgb", "hls", "hsv"])
    if mode == "rgb":
        return r*255., g*255., b*255.
    elif mode in ["hls", "hsv"]:
        return r*360., g*100., b*100.

def _normalise(r, g, b, mode="rgb"):
    check_param_in_list(mode, ["rgb", "hls", "hsv"])
    if mode == "rgb":
        return r/255., g/255., b/255.
    elif mode in ["hls", "hsv"]:
        return r/360., g/100., b/100.

def to_intensity(n):
    """Return intensity

    :param n: value between 0 and 1
    :return: value between 0 and 255; round(n*127.5+127.5)
    """
    check_range(n, 0, 1)
    return int(round(n * 127.5 + 127.5))

class HEX(object):
    """Class to check the validity of an hexadecimal string and get standard string

    By standard, we mean #FFFFFF (6 digits)

    ::

        >>> h = HEX()
        >>> h.is_valid_hex_color("#FFFF00")
        True

    """
    def __init__(self):
        pass

    def is_valid_hex_color(self, value, verbose=True):
        """Return True is the string can be interpreted as hexadecimal color

        Valid formats are

         * #FFF
         * #0000FF
         * 0x0000FF
         * 0xFA1
        """
        try:
            self.get_standard_hex_color(value)
            return True
        except Exception as err:
            if verbose:
                print(err)
            return False

    def get_standard_hex_color(self, value):
        """Return standard hexadecimal color

        By standard, we mean a string that starts with # sign followed by 6
        character, e.g. #AABBFF
        """
        if isinstance(value, str)==False:
            raise TypeError("value must be a string")
        if len(value) <= 3:
            raise ValueError("input string must be of type 0xFFF, 0xFFFFFF or #FFF or #FFFFFF")

        if value.startswith("0x") or value.startswith("0X"):
            value =  value[2:]
        elif value.startswith("#"):
            value = value[1:]
        else:
            raise ValueError("hexa string must start with a '#' sign or '0x' string")
        value = value.upper()
        # Now, we have either FFFFFF or FFF
        # now check the length
        for x in value:
            if x not in "0123456789ABCDEF":
                raise ValueError("Found invalid hexa character {0}".format(x))

        if len(value) == 6 or len(value) == 8:
            value  = "#" + value[0:6]
        elif len(value) == 3:
            value = "#" + value[0]*2 + value[1]*2 + value[2]*2
        else:
            raise ValueError("hexa string should be 3, 6 or 8 digits. if 8 digits, last 2 are ignored")
        return value

class Color(HEX):
    """Class to ease manipulation and conversion between color codes

    You can create an instance in many differen ways. You can either use a
    human-readable name as long as it is part of the
    `XFree86 list <http://en.wikipedia.org/wiki/X11_color_names>`_
    You can also provide a hexadecimal string (either 3 or 6 digits). You can
    use triplets of values corresponding to the RGB, HSV or HLS conventions.

    Here are some examples:

    .. doctest::

        from colormap import Color

        Color("red")           # human XFree86 compatible representation
        Color("#f00")          # standard 3 hex digits
        Color("#ff0000")       # standard 6 hex digits
        Color(hsv=(0,1,0.5))
        Color(hls=(0, 1, 0.5)) # HLS triplet
        Color(rgb=(1, 0, 0))   # RGB triplet
        Color(Color("red"))    # using an instance of :class:`Color`

    Note that the RGB, HLS and HSV triplets use normalised values. If you need
    to normalise the triplet, you can use :mod:`colormap.colors._normalise` that
    provides a function to normalise RGB, HLS and HSV triplets::

        colors._normalise(*(255, 255, 0), mode="rgb")
        colors._normalise(*(360, 50, 100), mode="hls")

    If you provide a string, it has to be a valid string from XFree86.
    In addition to the official names, the lower case names are valid. Besides,
    there are names with spaces. The equivalent names without space are also
    valid. Therefore the name "Spring Green", which is an official name can be
    provided as "Spring Green", "spring green", "springgreen" or "SpringGreen".

    """
    # Get official color names
    colors = XFree86_colors.copy()
    # add color names without spaces
    aliases = dict([(x.replace(" ", ""),x) for x in colors.keys() if " " in x])
    # add color names without spaces in lower cases
    aliases.update([(x.replace(" ", "").lower(),x) for x in colors.keys() if " " in x])
    # add color names in lower case
    aliases.update(dict([(x.lower(),x) for x in colors.keys()]))
    aliases.update(dict([(x,x) for x in colors.keys()]))

    # keep track of all possible names
    color_names = sorted(list(set(list(colors.keys()) +list( aliases.keys()))))

    def __init__(self, name=None, rgb=None, hls=None, hsv=None):
        super(Color, self).__init__()
        self._name = None
        self._mode = None
        self._rgb = None

        # Does the user provided the name argument (first one) as a string ?
        if isinstance(name, str):
            # if so, it can be a valid human name (e.g., red) or an hex
            # assuming that valid hexadecimal starts with # or 0x,
            # if we can interpret the string as an hexadecimal, we are done
            if self.is_valid_hex_color(name, verbose=False):
                self.hex = name
            else:
                # if not, then, the user probably provided a valid color name
                # the property will check the validity.
                self.name = name[:]
                #all other input parameters are ignored
        elif name == None:
            if rgb:
                self.rgb = rgb
            elif hls:
                self.hls = hls
            elif hsv:
                self.hsv = hsv
            else:
                raise ValueError("You must set one of the parameter")
        elif isinstance(name, Color):
            self.rgb = name.rgb
        else:
            raise ValueError("name parameter must be a string")

    def _get_name(self):
        return self._name
    def _set_name(self, name):
        check_param_in_list(name, self.color_names)
        name = self.aliases[name]
        self._name = name
        # set hex and rgb at the same time based on the name
        self.hex = self.colors[name]
    name = property(_get_name, _set_name)
    color = property(_get_name, _set_name)

    def _get_hex(self):
        return self._hex
    def _set_hex(self, value):
        # hex is an approximation made of 255 bits so do not define rgb here
        if self.is_valid_hex_color(value):
            value = self.get_standard_hex_color(value)
            self._hex = value
            if self._hex in self.colors.values():
                self._name = swapdict(self.colors, check_ambiguity=False)[self._hex]
            else:
                self._name = "undefined"
            self._rgb = hex2rgb(self._hex, normalise=True)
        else:
            # just to warn the user
            self.get_standard_hex_color(value)
    hex = property(_get_hex, _set_hex,
            doc="getter/setter the hexadecimal value.")

    def _get_rgb(self):
        return self._rgb
    def _set_rgb(self, value):
        # set name, hex and rgb
        self.hex = rgb2hex(*value , normalised=True)
        # must reset rgb with its real value (set_hex may round the rgb)
        # in _set_hex
        self._rgb = value
    rgb = property(_get_rgb, _set_rgb,
            doc="getter/setter the RGB values (3-length tuple)")

    def _get_hsv(self):
        hsv = rgb2hsv(*self.rgb)
        return hsv
    def _set_hsv(self, value):
        # TODO: value must be normalised
        self.rgb = hsv2rgb(*value)
    hsv = property(_get_hsv, _set_hsv,
            doc="getter/setter the HSV values (3-length tuple)")

    def _get_hls(self):
        hls = rgb2hls(*self.rgb)
        return hls
    def _set_hls(self, value):
        #hls = _normalise(*value, mode="hls")
        #else:
        hls = value
        self.rgb = hls2rgb(*hls)
    hls = property(_get_hls, _set_hls,
            doc="getter/setter the HLS values (3-length tuple)")

    def _get_lightness(self):
        return self.hls[1]
    def _set_lightness(self, lightness):
        h, l, s = self.hls
        self.hls = (h, lightness, s)
    lightness = property(_get_lightness, _set_lightness,
            doc="getter/setter the lightness in the HLS triplet")

    def _get_saturation_hls(self):
        return self.hls[2]
    def _set_saturation_hls(self, saturation):
        h, l, s = self.hls
        self.hls = (h, l, saturation)
    saturation_hls = property(_get_saturation_hls, _set_saturation_hls,
            doc="getter/setter the saturation in the HLS triplet")

    def _get_hue(self):
        return self.hls[0]
    def _set_hue(self, hue):
        h, l, s = self.hls
        self.hls = (hue, l, s)
    hue = property(_get_hue, _set_hue,
            doc="getter/setter the saturation in the HLS triplet")

    def _get_red(self):
        return self.rgb[0]
    def _set_red(self, red):
        r, g, b = self.rgb
        self.rgb = (red,g,b)
    red = property(_get_red, _set_red,
            doc="getter/setter for the red color in RGB triplet")

    def _get_green(self):
        return self.rgb[1]
    def _set_green(self, green):
        r, g, b = self.rgb
        self.rgb = (r, green, b)
    green = property(_get_green, _set_green,
            doc="getter/setter for the green color in RGB triplet")

    def _get_blue(self):
        return self.rgb[2]
    def _set_blue(self, blue):
        r, g, b = self.rgb
        self.rgb = (r, g, blue)
    blue = property(_get_blue, _set_blue,
            doc="getter/setter for the blue color in RGB triplet")

    def _get_value(self):
        return self.hls[0]
    def _set_value(self, value):
        h, s, v = self.hsv
        self.hsv = (h, s, value)
    value = property(_get_value, _set_value,
            doc="getter/setter the value in the HSV triplet")

    def _get_yiq(self):
        return colorsys.rgb_to_yiq(*self.rgb)
    yiq = property(_get_yiq, doc="Getter for the YIQ triplet")

    def __str__(self):
        txt = 'Color {0}\n'.format(self.name)
        txt+= '  hexa code: {0}\n'.format(self.hex)
        txt+= '  RGB code: {0}\n'.format(self.rgb)
        txt+= '  RGB code (un-normalised): {0}\n\n'.format([x*255 for x in self.rgb])
        txt+= '  HSV code: {0}\n'.format(self.hsv)
        txt+= '  HSV code: (un-normalised) {0} {1} {2}\n\n'.format(self.hsv[0]*360, self.hsv[1]*100, self.hsv[2]*100)
        txt+= '  HLS code: {0}\n'.format(self.hls)
        txt+= '  HLS code: (un-normalised) {0} {1} {2}\n\n'.format(self.hls[0]*360, self.hls[1]*100, self.hls[2]*100)
        return txt

class Colormap(object):
    """Class to create matplotlib colormap

    This example show how to get the pre-defined colormap called *heat*

    .. plot::
        :include-source:

        from pylab import *
        from colormap.colors import Colormap

        c = Colormap()
        cmap = c.get_cmap_heat()
        c.test_colormap(cmap)

    You may be more interested in building your own colormap::

        # design your own colormap
        d = {'blue': [0,0,0,1,1,1,0],
                'green':[0,1,1,1,0,0,0],
                'red':  [1,1,0,0,0,1,1]}
        cmap = c.cmap(d, reverse=False)

        # see the results
        c.test_colormap(cmap)

    If you want a simple linear colormap, you can use the example above,
    or use the :meth:`cmap_linear`. For instance for a diverging colormap
    from red to green (with with color in between)::

        cmap = c.cmap_linear("red", "white", "green")
        c.test_colormap(cmap)

    Even simpler, you can use a bicolor colormap :meth:`cmap_bicolor`. For instance
    for a red to green colormap::

        cmap = c.cmap_bicolor("red", "green")
        c.test_colormap(cmap)

    From matplotlib documentation, colormaps falls into 4 categories:

    #. Sequential schemes for unipolar data that progresses from low to high
    #. Diverging schemes for bipolar data that emphasizes positive or
       negative deviations from acentral value
    #. Cyclic schemes  meant for plotting values that wrap around at the
       endpoints, such as phase angle, wind direction, or time of day
    #. Qualitative schemes for nominal data that has no inherent ordering,
       where color is used only to distinguish categories

    :references: matplotlib documentation and examples
        http://matplotlib.org/examples/color/colormaps_reference.html
    """
    def _get_colormap_mpl(self):
        try:
            from matplotlib.pyplot import colormaps as _cmaps
            return _cmaps()
        except:
            return []
    colormaps = property(_get_colormap_mpl)

    def _get_sequentials(self):
        return ['Blues', 'BuGn', 'BuPu', 'GnBu', 'Greens', 'Greys', 'OrRd',
                'Oranges', 'PuBu', 'PuBuGn', 'PuRd', 'Purples', 'RdPu',
                'Reds', 'YlGn', 'YlGnBu', 'YlOrBr', 'YlOrRd']
    sequentials = property(_get_sequentials)

    def _get_sequentials2(self):
        return ['afmhot', 'autumn', 'bone', 'cool', 'copper',
               'gist_heat', 'gray', 'hot', 'pink',
               'spring', 'summer', 'winter']
    sequentials2 = property(_get_sequentials2)

    def _get_diverging(self):
         return ['BrBG',  'PRGn',  'PiYG', 'PuOr', 'RdBu', 'RdGy', 'RdYlBu',
                 'RdYlGn', 'Spectral', 'bwr', 'coolwarm', 'seismic']
    diverging = property(_get_diverging)

    def _get_diverging_black(self):
         return ['red_black_sky', 'red_black_blue', 'red_black_green', 'yellow_black_blue',
                 'yellow_black_sky', 'red_black_orange', 'pink_black_green(w3c)'
                 ]
    diverging_black = property(_get_diverging_black)

    def _get_qualitative(self):
        return ['Accent', 'Dark2', 'Paired', 'Pastel1', 'Pastel2',
                'Set1', 'Set2', 'Set3']
    qualitative = property(_get_qualitative)

    def _get_misc(self):
        return ['gist_earth', 'terrain', 'ocean', 'gist_stern',
                'brg', 'CMRmap', 'cubehelix', 'gnuplot', 'gnuplot2', 'gist_ncar',
                'nipy_spectral', 'jet', 'rainbow', 'gist_rainbow', 'hsv', 'flag', 'prism']
    misc = property(_get_misc)

    def plot_rgb_from_hex_list(self, cols):
        """This functions takes a list of hexadecimal values and plots
        the RGB curves. This can be handy to figure out the RGB functions
        to be used in the :meth:`get_cmap`.

        .. plot::
            :include-source:
            :width: 60%

            from colormap.colors import Colormap
            c = Colormap()
            t = ['#FF0000FF', '#FF4D00FF', '#FF9900FF', '#FFE500FF',
                 '#CCFF00FF', '#80FF00FF', '#33FF00FF', '#00FF19FF',
                 '#00FF66FF', '#00FFB2FF', '#00FFFFFF', '#00B3FFFF',
                 '#0066FFFF', '#001AFFFF', '#3300FFFF', '#7F00FFFF',
                 '#CC00FFFF','#FF00E6FF','#FF0099FF', '#FF004DFF']
            c.plot_rgb_from_hex_list(t)

        """
        import pylab
        red = [hex2rgb(x)[0]/255. for x in cols]
        blue = [hex2rgb(x)[2]/255. for x in cols]
        green = [hex2rgb(x)[1]/255. for x in cols]
        x = pylab.linspace(0, 1, len(cols))
        pylab.clf()
        pylab.plot(x, red, 'ro-', alpha=0.5)
        pylab.plot(x, green, 'gs-', alpha=0.5, markersize=15)
        pylab.plot(x, blue, 'bx-', alpha=0.5, markersize=15)
        pylab.ylim([-0.1, 1.1])

    def cmap_bicolor(self, color1, color2, reverse=False, N=256):
        """Provide 3 colors in format accepted by :class:`Color`

        ::

            >>> red = Color('red')
            >>> white = Color('white')
            >>> cmap = cmap_bicolor(red, white)

        """
        c1 = Color(color1)
        c2 = Color(color2)
        dico = {'red': [c1.red, c2.red],
                'green':[c1.green, c2.green],
                'blue':[c1.blue, c2.blue]}
        return self.cmap(dico, reverse=reverse, N=N)

    def cmap_linear(self, color1, color2, color3, reverse=False, N=256):
        """Provide 3 colors in format accepted by :class:`Color`

        ::

            red = Color('red')
            cmap = cmap_linear(red, 'white', '#0000FF')

        """
        c1 = Color(color1)
        c2 = Color(color2)
        c3 = Color(color3)
        dico = {'red': [c1.red, c2.red, c3.red],
                'green':[c1.green, c2.green, c3.green],
                'blue':[c1.blue, c2.blue, c3.blue]}

        return self.cmap(dico, reverse=reverse, N=N)

    def cmap(self, colors=None, reverse=False, N=256):
        """Return a colormap object to be used within matplotlib

        :param dict colors: a dictionary that defines the RGB colors to be
            used in the colormap. See :meth:`get_cmap_heat` for an example.
        :param bool reverse: reverse the colormap is  set to True (defaults to False)
        :param int N: Defaults to 50

        """
        # matplotlib colormaps
        if colors in self.colormaps:
            if reverse and colors.endswith("_r") is False:
                colors += "_r"
            from matplotlib.cm import get_cmap
            return get_cmap(colors)
        # custom ones
        elif colors in self.diverging_black:
            c1, c2, c3 = colors.split("_")
            # special case of sky, which does not exists
            c3 = c3.replace("sky", "deep sky blue")
            return self.cmap_linear(c1, c2, c3)
        elif colors == 'heat':
            return self.get_cmap_heat()
        elif colors == 'heat_r':
            return self.get_cmap_heat_r()

        # Keep these dependencies inside the function to allow
        # installation of colormap without those dependencies
        # FIXME remove numpy dependencies
        import numpy as np
        # extracted from R, heat.colors(20)

        if reverse:
            for k in colors.keys():
                colors[k].reverse()

        # If index not given, RGB colors are evenly-spaced in colormap.
        index = np.linspace(0, 1, len(colors['red']))

        # Adapt color_data to the form expected by LinearSegmentedColormap.
        color_data = dict((key, [(x, y, y) for x, y in zip(index, value)])
                 for key, value in list(colors.items()))

        import matplotlib
        f = matplotlib.colors.LinearSegmentedColormap
        m = f('my_color_map', color_data, N)
        return m

    def get_cmap_heat(self):
        """Return a heat colormap matplotlib-compatible colormap

        This heat colormap should be equivalent to heat.colors() in R.

        ::

            >>> from colormap.colors import Colormap
            >>> cmap = Colormap.get_cmap_heat()

        You can generate the colormap based solely on this information for the RGB
        functions along::

            d=  {   'blue':[0,0,0,0,1],
                    'green':[0,.35,.7,1,1],
                    'red':[1,1,1,1,1]}
            cmap = Colormap.get_cmap(d)

        """
        return self.cmap(
                {   'blue':[0, 0, 0, 0, 1],
                    'green':[0, .35, .7, 1, 1],
                    'red':[1, 1, 1, 1, 1]}, reverse=False)

    def get_cmap_heat_r(self):
        """Return a heat colormap matplotlib-compatible colormap

        Same as :meth:`get_cmap_heat` but reversed
        """
        return self.cmap(
                {   'blue':[0, 0, 0, 0, 1],
                    'green':[0, .35, .7, 1, 1],
                    'red':[1, 1, 1, 1, 1]}, reverse=True)

    def get_cmap_rainbow(self):
        """colormap similar to rainbow colormap from R

        .. note:: The red is actually appearing on both sides... Yet
            this looks like what is coded in R 3.0.1

        """
        return self.cmap(
                {   'blue': [0, 0, 0, 1, 1, 1, 0],
                    'green':[0, 1, 1, 1, 0, 0, 0],
                    'red':  [1, 1, 0, 0, 0, 1, 1]}, reverse=False)

    def get_cmap_red_green(self):
        return self.cmap(
                {   'green': [0, 0.4, 0.6, .75, .8, .9, 1, .9, .8, .6],
                    'blue' : [0, .4, .6, .75, .8, .7, .6, .35, .17, .1],
                    'red':   [1, 1, 1, 1, 1, .9, .8, .6, .3, .1]}, reverse=True)

    def test_colormap(self, cmap=None):
        """plot one colormap for testing

        By default, test the :meth:`get_cmap_heat`

        """
        if cmap is None:
            cmap = self.get_cmap_heat()
        import numpy as np
        from pylab import clf, pcolor, colorbar, show, linspace, axis
        A, B = np.meshgrid(linspace(0, 10, 100), linspace(0, 10, 100))
        clf()
        pcolor((A-5)**2+(B-5)**2, cmap=cmap)
        colorbar()
        show()
        axis('off')

    def plot_colormap(self, cmap_list=None):
        """cmap_list list of valid cmap or name of a set (sequential,
        diverging,)

        if none, plot all known colors

        .. .. plot::
        ..    :width:80%
        ..    :include-source:

        ..    from colormap import Colormap
        ..    c = Colormap()
        ..    c.plot_colormap('sequential')

        """
        from pylab import subplots

        if isinstance(cmap_list, str):
            if cmap_list in ['sequentials','sequentials2','qualitative',
                             'misc','diverging', 'diverging_black']:
                cmap_list = getattr(self, cmap_list)
            else:
                cmap_list = [cmap_list]
        if isinstance(cmap_list, list) is not True:
            raise TypeError("""input must be a list of srtings or a single string. Each string should be found. For a user-defined cmap, use test_colormap""")
        for this in cmap_list:
            if this not in self.colormaps and this not in self.diverging_black:
                raise ValueError("unknown colormap name. Please check valid names in colormaps attribute")

        nrows = len(cmap_list)

        gradient = [x/255. for x in range(0,256)]
        gradient = [gradient, gradient]
        #np.vstack((gradient, gradient))

        fig, axes = subplots(nrows=nrows)
        fig.subplots_adjust(top=0.95, bottom=0.05, left=0.05, right=0.8)

        for ax, name in zip(axes, cmap_list):
            ax.imshow(gradient, aspect='auto', cmap=self.cmap(name))
            pos = list(ax.get_position().bounds)
            x_text = pos[2] + 0.08
            y_text = pos[1] + pos[3]/2.
            fig.text(x_text, y_text, name, va='center', ha='left', fontsize=10)

        # Turn off *all* ticks & spines, not just the ones with colormaps.
        for ax in axes:
            ax.set_axis_off()


import scipy.io
from sklearn.metrics import accuracy_score
from sklearn import cross_validation
from sklearn import svm
from skfeature.function.statistical_based import chi_square

def main():
    # load data
    mat = scipy.io.loadmat('../data/BASEHOCK.mat')
    X = mat['X']    # data
    X = X.astype(float)
    y = mat['Y']    # label
    y = y[:, 0]
    n_samples, n_features = X.shape    # number of samples and number of features

    # split data into 10 folds
    ss = cross_validation.KFold(n_samples, n_folds=10, shuffle=True)

    # perform evaluation on classification task
    num_fea = 100    # number of selected features
    clf = svm.LinearSVC()    # linear SVM

    correct = 0
    for train, test in ss:
        # obtain the chi-square score of each feature
        score = chi_square.chi_square(X, y)

        # rank features in descending order according to score
        idx = chi_square.feature_ranking(score)

        # obtain the dataset on the selected features
        selected_features = X[:, idx[0:num_fea]]

        # train a classification model with the selected features on the training dataset
        clf.fit(selected_features[train], y[train])

        # predict the class labels of test data
        y_predict = clf.predict(selected_features[test])

        # obtain the classification accuracy on the test data
        acc = accuracy_score(y[test], y_predict)
        correct = correct + acc

    # output the average classification accuracy over all 10 folds
    print 'Accuracy:', float(correct)/10

if __name__ == '__main__':
    main()


import jieba
import pandas as pd
from gensim import corpora, models, similarities

# 训练样本
raw_documents = [
    '0无偿居间介绍买卖毒品的行为应如何定性',
    '1吸毒男动态持有大量毒品的行为该如何认定',
    '2如何区分是非法种植毒品原植物罪还是非法制造毒品罪',
    '3为毒贩贩卖毒品提供帮助构成贩卖毒品罪',
    '4将自己吸食的毒品原价转让给朋友吸食的行为该如何认定',
    '5为获报酬帮人购买毒品的行为该如何认定',
    '6毒贩出狱后再次够买毒品途中被抓的行为认定',
    '7虚夸毒品功效劝人吸食毒品的行为该如何认定',
    '8妻子下落不明丈夫又与他人登记结婚是否为无效婚姻',
    '9一方未签字办理的结婚登记是否有效',
    '10夫妻双方1990年按农村习俗举办婚礼没有结婚证 一方可否起诉离婚',
    '11结婚前对方父母出资购买的住房写我们二人的名字有效吗',
    '12身份证被别人冒用无法登记结婚怎么办？',
    '13同居后又与他人登记结婚是否构成重婚罪',
    '14未办登记只举办结婚仪式可起诉离婚吗',
    '15同居多年未办理结婚登记，是否可以向法院起诉要求离婚'
]

def main():
    corpora_documents = []
    for item_text in raw_documents:
        item_str = list(jieba.cut(item_text))
        corpora_documents.append(item_str)

    dictionary = corpora.Dictionary(corpora_documents)
    corpus = [dictionary.doc2bow(text) for text in corpora_documents]

    similarity =similarities.Similarity('-Similarity-index', corpus, num_features=400)

    test_data_1 = '你好，我想问一下我想离婚他不想离，孩子他说不要，是六个月就自动生效离婚'
    test_cut_raw_1 = jieba.cut(test_data_1)
    test_corpus_1 = dictionary.doc2bow(test_cut_raw_1)
    similarity.num_best = 5
    # 返回最相似的样本材料,(index_of_document, similarity) tuples
    print(similarity[test_corpus_1])

if __name__ == '__main__':
    main()


#
# Jupyter notebook testing logic adapted from
#   https://gist.github.com/lheagy/f216db7220713329eb3fc1c2cd3c7826
#
# The MIT License (MIT)
#
# Copyright (c) 2016 Lindsey Heagy
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# Raw

import unittest
import sys
import os
import subprocess
try:
    import jupyter
    jupyter_available = True
except:
    jupyter_available = False
try:
    import pandas
    pandas_available = True
except:
    pandas_available = False
try:
    import networkx
    networkx_available = True
except:
    networkx_available = False

timeout=120

requires_pandas = set(['max_flow_interdict', 'min_cost_flow_interdict', 'multi_commodity_flow_interdict', 'sp_interdict', 'min_cost_flow', 'mst'])
requires_networkx = set(['mst'])

# Testing for the notebooks - use nbconvert to execute all cells of the
# notebook

# For testing on TravisCI, be sure to include a requirements.txt that
# includes jupyter so that you run on the most up-to-date version.

# Where are the notebooks?
TESTDIR = os.path.dirname(os.path.abspath(__file__))
#NBDIR = os.path.sep.join(TESTDIR.split(os.path.sep)[:-2] + ['notebooks/']) # where are the notebooks?

def setUp():
    nbpaths = []  # list of notebooks, with file paths
    nbnames = []  # list of notebook names (for making the tests)

    print(TESTDIR)
    # walk the test directory and find all notebooks
    for dirname, dirnames, filenames in os.walk(TESTDIR):
        for filename in filenames:
            if filename.endswith('.ipynb') and not filename.endswith('-checkpoint.ipynb'):
                nbpaths.append(os.path.abspath(dirname) + os.path.sep + filename) # get abspath of notebook
                nbnames.append(''.join(filename[:-6])) # strip off the file extension
    return nbpaths, nbnames

def get(nbname, nbpath):

    # use nbconvert to execute the notebook
    def test_func(self):
        print('\n--------------- Testing {0} ---------------'.format(nbname))
        print('   {0}'.format(nbpath))
        if not jupyter_available:
            self.skipTest("Jupyter unavailable")
        if nbname in requires_pandas and not pandas_available:
            self.skipTest("Pandas unavailable")
        if nbname in requires_networkx and not networkx_available:
            self.skipTest("Networkx unavailable")

        # execute the notebook using nbconvert to generate html
        dir_=os.path.dirname(nbpath)
        os.chdir(dir_)
        nbexe = subprocess.Popen(
            [ 'jupyter', 'nbconvert', '{0}'.format(nbpath),
              '--execute',
              '--inplace',
              '--ExecutePreprocessor.kernel_name=python%s' % (
                  {2:"",3:"3"}[sys.version_info[0]], ),
              '--ExecutePreprocessor.timeout='+str(timeout)],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        output, err = nbexe.communicate()
        check = nbexe.returncode
        if check == 0:
            print('\n ..... {0} Passed ..... \n'.format(nbname))
            # if passed remove the generated html file
            #subprocess.call(['rm', '{0}.html'.format( os.path.sep.join(os.getcwd().split(os.path.sep) + [nbpath.split(os.path.sep)[-1][:-6]]))])
        else:
            print('\n <<<<< {0} FAILED >>>>> \n'.format(nbname))
            print('Captured Output: \n {0}'.format(err))

        self.assertEqual(check, 0)

    return test_func

class TestNotebooks(unittest.TestCase):
    pass

nbpaths, nbnames = setUp()
# Check for duplicates
tmp = set()
for name in nbnames:
    if name in tmp:
        raise IOError("ERROR: duplicate test name %s" % name)
    tmp.add(name)

# build test for each notebook
for i, nb in enumerate(nbnames):
    #print((i,nb,nbpaths[i]))
    setattr(TestNotebooks, 'test_'+nb, get(nb, nbpaths[i]))

if __name__ == '__main__':
    unittest.main()


#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""Example of using Estimator with multiple GPUs to distribute one model.

This example only runs if you have multiple GPUs to assign to.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
from sklearn import datasets
from sklearn import metrics
from sklearn import model_selection
import tensorflow as tf

X_FEATURE = 'x'  # Name of the input feature.

def my_model(features, labels, mode):
  """DNN with three hidden layers, and dropout of 0.1 probability.

  Note: If you want to run this example with multiple GPUs, Cuda Toolkit 7.0 and
  CUDNN 6.5 V2 from NVIDIA need to be installed beforehand.

  Args:
    features: Dict of input `Tensor`.
    labels: Label `Tensor`.
    mode: One of `ModeKeys`.

  Returns:
    `EstimatorSpec`.
  """
  # Create three fully connected layers respectively of size 10, 20, and 10 with
  # each layer having a dropout probability of 0.1.
  net = features[X_FEATURE]
  with tf.device('/gpu:1'):
    for units in [10, 20, 10]:
      net = tf.layers.dense(net, units=units, activation=tf.nn.relu)
      net = tf.layers.dropout(net, rate=0.1)

  with tf.device('/gpu:2'):
    # Compute logits (1 per class).
    logits = tf.layers.dense(net, 3, activation=None)

    # Compute predictions.
    predicted_classes = tf.argmax(logits, 1)
    if mode == tf.estimator.ModeKeys.PREDICT:
      predictions = {
          'class': predicted_classes,
          'prob': tf.nn.softmax(logits)
      }
      return tf.estimator.EstimatorSpec(mode, predictions=predictions)

    # Convert the labels to a one-hot tensor of shape (length of features, 3)
    # and with a on-value of 1 for each one-hot vector of length 3.
    onehot_labels = tf.one_hot(labels, 3, 1, 0)
    # Compute loss.
    loss = tf.losses.softmax_cross_entropy(
        onehot_labels=onehot_labels, logits=logits)

    # Create training op.
    if mode == tf.estimator.ModeKeys.TRAIN:
      optimizer = tf.train.AdagradOptimizer(learning_rate=0.1)
      train_op = optimizer.minimize(
          loss, global_step=tf.train.get_global_step())
      return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)

    # Compute evaluation metrics.
    eval_metric_ops = {
        'accuracy': tf.metrics.accuracy(
            labels=labels, predictions=predicted_classes)
    }
    return tf.estimator.EstimatorSpec(
        mode, loss=loss, eval_metric_ops=eval_metric_ops)

def main(unused_argv):
  iris = datasets.load_iris()
  x_train, x_test, y_train, y_test = model_selection.train_test_split(
      iris.data, iris.target, test_size=0.2, random_state=42)

  classifier = tf.estimator.Estimator(model_fn=my_model)

  # Train.
  train_input_fn = tf.estimator.inputs.numpy_input_fn(
      x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True)
  classifier.train(input_fn=train_input_fn, steps=100)

  # Predict.
  test_input_fn = tf.estimator.inputs.numpy_input_fn(
      x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False)
  predictions = classifier.predict(input_fn=test_input_fn)
  y_predicted = np.array(list(p['class'] for p in predictions))
  y_predicted = y_predicted.reshape(np.array(y_test).shape)

  # Score with sklearn.
  score = metrics.accuracy_score(y_test, y_predicted)
  print('Accuracy (sklearn): {0:f}'.format(score))

  # Score with tensorflow.
  scores = classifier.evaluate(input_fn=test_input_fn)
  print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy']))

if __name__ == '__main__':
  tf.app.run()


import numpy as np
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from numpy.testing import assert_array_almost_equal

from skgarden.quantile import DecisionTreeQuantileRegressor
from skgarden.quantile import ExtraTreeQuantileRegressor
from skgarden.quantile.utils import weighted_percentile

boston = load_boston()
X, y = boston.data, boston.target
X_train, X_test, y_train, y_test = train_test_split(
    X, y, train_size=0.6, test_size=0.4, random_state=0)
X_train = np.array(X_train, dtype=np.float32)
X_test = np.array(X_test, dtype=np.float32)
estimators = [
    DecisionTreeQuantileRegressor(random_state=0),
    ExtraTreeQuantileRegressor(random_state=0)
]

def test_quantiles():
    # Test with max depth 1.
    for est in estimators:
        est.set_params(max_depth=1)
        est.fit(X_train, y_train)
        tree = est.tree_

        for q in [20, 40, 50, 60, 80, 90]:
            left_ind = X_train[:, tree.feature[0]] <= tree.threshold[0]
            right_ind = X_train[:, tree.feature[0]] > tree.threshold[0]

            # fixme
            left_q = weighted_percentile(y_train[left_ind], q)
            right_q = weighted_percentile(y_train[right_ind], q)

            for curr_X, curr_y in [[X_train, y_train], [X_test, y_test]]:
                actual_q = np.zeros(curr_X.shape[0])
                left_ind = curr_X[:, tree.feature[0]] <= tree.threshold[0]
                actual_q[left_ind] = left_q
                right_ind = curr_X[:, tree.feature[0]] > tree.threshold[0]
                actual_q[right_ind] = right_q

                expected_q = est.predict(curr_X, quantile=q)
                assert_array_almost_equal(expected_q, actual_q)

def test_max_depth_None():
    # Since each leaf is pure and has just one unique value.
    # the mean equals any quantile.
    for est in estimators:
        est.set_params(max_depth=None)
        est.fit(X_train, y_train)

        for quantile in [20, 40, 50, 60, 80, 90]:

            for curr_X in [X_train, X_test]:
                assert_array_almost_equal(
                    est.predict(curr_X, quantile=None),
                    est.predict(curr_X, quantile=quantile), 1)

def test_tree_toy_data():
    rng = np.random.RandomState(0)
    x1 = rng.randn(1, 10)
    X1 = np.tile(x1, (10000, 1))
    x2 = 20.0 * rng.randn(1, 10)
    X2 = np.tile(x2, (10000, 1))
    X = np.vstack((X1, X2))

    y1 = rng.randn(10000)
    y2 = 5.0 + rng.randn(10000)
    y = np.concatenate((y1, y2))

    for est in estimators:
        est.set_params(max_depth=1)
        est.fit(X, y)
        for quantile in [20, 30, 40, 50, 60, 70, 80]:
            assert_array_almost_equal(
                est.predict(x1, quantile=quantile),
                [np.percentile(y1, quantile)], 3)
            assert_array_almost_equal(
                est.predict(x2, quantile=quantile),
                [np.percentile(y2, quantile)], 3)


#!/usr/bin/env python3

"""
This is an real-time frequency meter of two PMUs.
This code connects to two PMUs, plot the frequency of the past 300 time-stamps and update the plot in real-time.
"""

from phasortoolbox import PDC,Client
import matplotlib.pyplot as plt
import numpy as np
import gc
import logging
logging.basicConfig(level=logging.DEBUG)

class FreqMeter(object):
    def __init__(self):
        x = np.linspace(-10.0, 0.0, num=300, endpoint=False)
        y = [60.0]*300

        plt.ion()
        self.fig = plt.figure()
        self.ax1 = self.fig.add_subplot(211)
        self.line1, = self.ax1.plot(x, y)
        plt.title('PMU1 Frequency Plot')
        plt.xlabel('Time (s)')
        plt.ylabel('Freq (Hz)')
        self.ax2 = self.fig.add_subplot(212)
        self.line2, = self.ax2.plot(x, y)
        plt.title('PMU2 Frequency Plot')
        plt.xlabel('Time (s)')
        plt.ylabel('Freq (Hz)')
        plt.tight_layout()

    def update_plot(self, synchrophasors):
        y_data = [[],[]]
        for synchrophasor in synchrophasors:
            for i, msg in enumerate(synchrophasor):
                y_data[i].append(msg.data.pmu_data[0].freq)

        self.line1.set_ydata(y_data[0])
        self.line2.set_ydata(y_data[1])
        self.ax1.set_ylim(min(y_data[0]),max(y_data[0]))
        self.ax2.set_ylim(min(y_data[1]),max(y_data[1]))
        self.fig.canvas.draw()
        self.fig.canvas.flush_events()
        del(synchrophasors)
        gc.collect()

if __name__ == '__main__':
    pmu_client1 = Client(remote_ip='10.0.0.1', remote_port=4722, idcode=1, mode='TCP')
    pmu_client2 = Client(remote_ip='10.0.0.2', remote_port=4722, idcode=2, mode='TCP')

    fm = FreqMeter()
    pdc = PDC(clients=[pmu_client1,pmu_client2],history=300)
    pdc.callback = fm.update_plot

    pdc.run()


import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import os

from pygeocoder import Geocoder

plt.ion()

year = 2015
record_num = []

aggregation_rule = {'Sum': sum}

ts_all = None

aggregation_window = "1min"
print " aggregate data at" + aggregation_window + "resolution"

for year in [2014, 2015]:
  for month in xrange(1, 13):
    datafileName = 'yellow_tripdata_' + str(year) + '-' + "{:0>2d}".format(month) + '.csv'

    if os.path.isfile(datafileName):
      print " Load Datafile: ", datafileName
      # df = pd.read_csv(datafileName, header=0, nrows=100,  usecols=[1, 3, 5, 6],
      #                  names=['pickup_datetime', 'passenger_count', 'pickup_longitude', 'pickup_latitude'])
      #
      # postcode = np.zeros(len(df))
      # for i in xrange(len(df)):
      #   try:
      #     results = Geocoder.reverse_geocode(df['pickup_latitude'][i], df['pickup_longitude'][i])
      #     postcode[i] = results.postal_code
      #   except:
      #     pass

      df = pd.read_csv(datafileName, header=0, usecols=[1, 3], names=['pickup_datetime', 'passenger_count'])

      record_num.append(len(df))

      ts = pd.Series(np.array(df.passenger_count), index=pd.to_datetime(df.pickup_datetime))
      del df

      ts_aggregate = ts.resample(aggregation_window, how=aggregation_rule)

      if ts_all is not None:
        print " concat ts_all"
        ts_all = pd.concat([ts_all, ts_aggregate])
      else:
        print " initialize ts_all"
        ts_all = ts_aggregate
    else:
      print datafileName, " not exist"

print "include time of day and day of week as input field"
date = ts_all.index
dayofweek = (date.dayofweek)
timeofday = (date.hour*60 + date.minute)
passenger_count = np.array(ts_all['Sum'])
seq = pd.DataFrame(np.transpose(np.array([passenger_count, timeofday, dayofweek])), columns=['passenger_count', 'timeofday', 'dayofweek'], index=ts_all.index)

plt.close('all')
plt.figure(1)
plt.plot(seq.index, seq.passenger_count)

import csv
outputFileName = "nyc_taxi_" + aggregation_window + ".csv"
outputFile = open(outputFileName,"w")
csvWriter = csv.writer(outputFile)
csvWriter.writerow(['timestamp', 'passenger_count', 'timeofday', 'dayofweek'])
csvWriter.writerow(['datetime', 'int', 'int', 'string'])
csvWriter.writerow(['T', '', '', ''])
for i in range(len(ts_all)):
  csvWriter.writerow([seq.index[i], seq.passenger_count[i], seq.timeofday[i], seq.dayofweek[i]])
outputFile.close()


"""Testing for Spectral Clustering methods"""

from sklearn.externals.six.moves import cPickle

dumps, loads = cPickle.dumps, cPickle.loads

import numpy as np
from scipy import sparse

from sklearn.utils import check_random_state
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_warns_message

from sklearn.cluster import SpectralClustering, spectral_clustering
from sklearn.cluster.spectral import spectral_embedding
from sklearn.cluster.spectral import discretize
from sklearn.metrics import pairwise_distances
from sklearn.metrics import adjusted_rand_score
from sklearn.metrics.pairwise import kernel_metrics, rbf_kernel
from sklearn.datasets.samples_generator import make_blobs

def test_spectral_clustering():
    S = np.array([[1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0],
                  [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0],
                  [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0],
                  [0.2, 0.2, 0.2, 1.0, 1.0, 1.0, 1.0],
                  [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0],
                  [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0],
                  [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0]])

    for eigen_solver in ('arpack', 'lobpcg'):
        for assign_labels in ('kmeans', 'discretize'):
            for mat in (S, sparse.csr_matrix(S)):
                model = SpectralClustering(random_state=0, n_clusters=2,
                                           affinity='precomputed',
                                           eigen_solver=eigen_solver,
                                           assign_labels=assign_labels
                                          ).fit(mat)
                labels = model.labels_
                if labels[0] == 0:
                    labels = 1 - labels

                assert_array_equal(labels, [1, 1, 1, 0, 0, 0, 0])

                model_copy = loads(dumps(model))
                assert_equal(model_copy.n_clusters, model.n_clusters)
                assert_equal(model_copy.eigen_solver, model.eigen_solver)
                assert_array_equal(model_copy.labels_, model.labels_)

def test_spectral_amg_mode():
    # Test the amg mode of SpectralClustering
    centers = np.array([
        [0., 0., 0.],
        [10., 10., 10.],
        [20., 20., 20.],
    ])
    X, true_labels = make_blobs(n_samples=100, centers=centers,
                                cluster_std=1., random_state=42)
    D = pairwise_distances(X)  # Distance matrix
    S = np.max(D) - D  # Similarity matrix
    S = sparse.coo_matrix(S)
    try:
        from pyamg import smoothed_aggregation_solver

        amg_loaded = True
    except ImportError:
        amg_loaded = False
    if amg_loaded:
        labels = spectral_clustering(S, n_clusters=len(centers),
                                     random_state=0, eigen_solver="amg")
        # We don't care too much that it's good, just that it *worked*.
        # There does have to be some lower limit on the performance though.
        assert_greater(np.mean(labels == true_labels), .3)
    else:
        assert_raises(ValueError, spectral_embedding, S,
                      n_components=len(centers),
                      random_state=0, eigen_solver="amg")

def test_spectral_unknown_mode():
    # Test that SpectralClustering fails with an unknown mode set.
    centers = np.array([
        [0., 0., 0.],
        [10., 10., 10.],
        [20., 20., 20.],
    ])
    X, true_labels = make_blobs(n_samples=100, centers=centers,
                                cluster_std=1., random_state=42)
    D = pairwise_distances(X)  # Distance matrix
    S = np.max(D) - D  # Similarity matrix
    S = sparse.coo_matrix(S)
    assert_raises(ValueError, spectral_clustering, S, n_clusters=2,
                  random_state=0, eigen_solver="<unknown>")

def test_spectral_unknown_assign_labels():
    # Test that SpectralClustering fails with an unknown assign_labels set.
    centers = np.array([
        [0., 0., 0.],
        [10., 10., 10.],
        [20., 20., 20.],
    ])
    X, true_labels = make_blobs(n_samples=100, centers=centers,
                                cluster_std=1., random_state=42)
    D = pairwise_distances(X)  # Distance matrix
    S = np.max(D) - D  # Similarity matrix
    S = sparse.coo_matrix(S)
    assert_raises(ValueError, spectral_clustering, S, n_clusters=2,
                  random_state=0, assign_labels="<unknown>")

def test_spectral_clustering_sparse():
    X, y = make_blobs(n_samples=20, random_state=0,
                      centers=[[1, 1], [-1, -1]], cluster_std=0.01)

    S = rbf_kernel(X, gamma=1)
    S = np.maximum(S - 1e-4, 0)
    S = sparse.coo_matrix(S)

    labels = SpectralClustering(random_state=0, n_clusters=2,
                                affinity='precomputed').fit(S).labels_
    assert_equal(adjusted_rand_score(y, labels), 1)

def test_affinities():
    # Note: in the following, random_state has been selected to have
    # a dataset that yields a stable eigen decomposition both when built
    # on OSX and Linux
    X, y = make_blobs(n_samples=20, random_state=0,
                      centers=[[1, 1], [-1, -1]], cluster_std=0.01
                     )
    # nearest neighbors affinity
    sp = SpectralClustering(n_clusters=2, affinity='nearest_neighbors',
                            random_state=0)
    assert_warns_message(UserWarning, 'not fully connected', sp.fit, X)
    assert_equal(adjusted_rand_score(y, sp.labels_), 1)

    sp = SpectralClustering(n_clusters=2, gamma=2, random_state=0)
    labels = sp.fit(X).labels_
    assert_equal(adjusted_rand_score(y, labels), 1)

    X = check_random_state(10).rand(10, 5) * 10

    kernels_available = kernel_metrics()
    for kern in kernels_available:
        # Additive chi^2 gives a negative similarity matrix which
        # doesn't make sense for spectral clustering
        if kern != 'additive_chi2':
            sp = SpectralClustering(n_clusters=2, affinity=kern,
                                    random_state=0)
            labels = sp.fit(X).labels_
            assert_equal((X.shape[0],), labels.shape)

    sp = SpectralClustering(n_clusters=2, affinity=lambda x, y: 1,
                            random_state=0)
    labels = sp.fit(X).labels_
    assert_equal((X.shape[0],), labels.shape)

    def histogram(x, y, **kwargs):
        # Histogram kernel implemented as a callable.
        assert_equal(kwargs, {})    # no kernel_params that we didn't ask for
        return np.minimum(x, y).sum()

    sp = SpectralClustering(n_clusters=2, affinity=histogram, random_state=0)
    labels = sp.fit(X).labels_
    assert_equal((X.shape[0],), labels.shape)

    # raise error on unknown affinity
    sp = SpectralClustering(n_clusters=2, affinity='<unknown>')
    assert_raises(ValueError, sp.fit, X)

def test_discretize(seed=8):
    # Test the discretize using a noise assignment matrix
    random_state = np.random.RandomState(seed)
    for n_samples in [50, 100, 150, 500]:
        for n_class in range(2, 10):
            # random class labels
            y_true = random_state.randint(0, n_class + 1, n_samples)
            y_true = np.array(y_true, np.float)
            # noise class assignment matrix
            y_indicator = sparse.coo_matrix((np.ones(n_samples),
                                             (np.arange(n_samples),
                                              y_true)),
                                            shape=(n_samples,
                                                   n_class + 1))
            y_true_noisy = (y_indicator.toarray()
                            + 0.1 * random_state.randn(n_samples,
                                                       n_class + 1))
            y_pred = discretize(y_true_noisy, random_state)
            assert_greater(adjusted_rand_score(y_true, y_pred), 0.8)


# -*- coding: utf-8 -*-
"""Common function for creating plots with matplotlib."""

__author__ = 'Patrick Michl'
__email__ = 'frootlab@gmail.com'
__license__ = 'GPLv3'
__docformat__ = 'google'

import numpy as np
from nemoa.types import OptDict

class Plot:
    """Base class for matplotlib plots.

    Export classes like Histogram, Heatmap or Graph share a common
    interface to matplotlib, as well as certain plotting attributes.
    This base class is intended to provide a unified interface to access
    matplotlib and those attributes.

    Attributes:

    """

    _default: dict = {
        'fileformat': 'pdf',
        'figure_size': (10.0, 6.0),
        'dpi': None,
        'bg_color': 'none',
        'usetex': False,
        'font_family': 'sans-serif',
        'style': 'seaborn-white',
        'title': None,
        'show_title': True,
        'title_fontsize': 14.0
    }

    _config: dict = {}
    _kwds: dict = {}

    _plt = None
    _fig = None
    _axes = None

    def __init__(self, **kwds):
        """ """
        try:
            import matplotlib
        except ImportError as err:
            raise ImportError(
                "requires package matplotlib: "
                "https://matplotlib.org") from err

        # merge config from defaults, current config and keyword arguments
        self._kwds = kwds
        self._config = {**self._default, **self._config, **kwds}

        # update global matplotlib settings
        matplotlib.rc('text', usetex=self._config['usetex'])
        matplotlib.rc('font', family=self._config['font_family'])

        # link matplotlib.pyplot
        import matplotlib.pyplot as plt
        self._plt = plt

        # close previous figures
        plt.close('all')

        # update plot settings
        plt.style.use(self._config['style'])

        # create figure
        self._fig = plt.figure(
            figsize=self._config['figure_size'],
            dpi=self._config['dpi'],
            facecolor=self._config['bg_color'])

        # create subplot (matplotlib.axes.Axes)
        self._axes = self._fig.add_subplot(111)

    def set_default(self, config: OptDict = None) -> bool:
        """Set default values."""
        self._config = {**self._config, **(config or {}), **self._kwds}

        return True

    def plot_title(self) -> bool:
        """Plot title."""
        if not self._config['show_title']:
            return False

        title = self._config['title'] or 'Unknown'
        fontsize = self._config['title_fontsize']

        getattr(self._plt, 'title')(title, fontsize=fontsize)
        return True

    def show(self) -> None:
        """Show plot."""
        getattr(self._plt, 'show')()

    def save(self, path, **kwds):
        """Save plot to file."""
        return self._fig.savefig(path, dpi=self._config['dpi'], **kwds)

    def release(self):
        """Clear current plot."""
        return self._fig.clear()

class Heatmap(Plot):
    """ """

    _config = {
        'interpolation': 'nearest',
        'grid': True
    }

    def plot(self, array):
        """ """

        try:
            from matplotlib.cm import hot_r
        except ImportError as err:
            raise ImportError(
                "requires package matplotlib: "
                "https://matplotlib.org") from err

        # plot grid
        self._axes.grid(self._config['grid'])

        # plot heatmap
        cax = self._axes.imshow(
            array,
            cmap=hot_r,
            interpolation=self._config['interpolation'],
            extent=(0, array.shape[1], 0, array.shape[0]))

        # create labels for axis
        max_font_size = 12.
        x_labels = []
        for label in self._config['x_labels']:
            if ':' in label:
                label = label.split(':', 1)[1]
            x_labels.append(get_texlabel(label))
        y_labels = []
        for label in self._config['y_labels']:
            if ':' in label:
                label = label.split(':', 1)[1]
            y_labels.append(get_texlabel(label))
        fontsize = min(max_font_size, \
            400. / float(max(len(x_labels), len(y_labels))))
        self._plt.xticks(
            np.arange(len(x_labels)) + 0.5,
            tuple(x_labels), fontsize=fontsize, rotation=65)
        self._plt.yticks(
            len(y_labels) - np.arange(len(y_labels)) - 0.5,
            tuple(y_labels), fontsize=fontsize)

        # create colorbar
        cbar = self._fig.colorbar(cax)
        for tick in cbar.ax.get_yticklabels():
            tick.set_fontsize(9)

        # (optional) plot title
        self.plot_title()

        return True

class Histogram(Plot):
    """ """

    _config = {
        'bins': 100,
        'facecolor': 'lightgrey',
        'edgecolor': 'black',
        'histtype': 'bar',
        'linewidth': 0.5,
        'grid': True
    }

    def plot(self, array):
        """ """

        # plot grid
        self._axes.grid(self._config['grid'])

        # plot histogram
        self._axes.hist(
            array,
            bins=self._config['bins'],
            facecolor=self._config['facecolor'],
            histtype=self._config['histtype'],
            linewidth=self._config['linewidth'],
            edgecolor=self._config['edgecolor'])

        # (optional) plot title
        self.plot_title()

        return True

class Scatter2D(Plot):
    """ """

    _config = {
        'grid': True,
        'pca': True
    }

    @staticmethod
    def _pca2d(array):
        """Calculate projection to largest two principal components."""
        # get dimension of array
        dim = array.shape[1]

        # calculate covariance matrix
        cov = np.cov(array.T)

        # calculate eigevectors and eigenvalues
        vals, vecs = np.linalg.eig(cov)

        # sort eigevectors by absolute eigenvalues
        pairs = [(np.abs(vals[i]), vecs[:, i]) for i in range(len(vals))]
        pairs.sort(key=lambda x: x[0], reverse=True)

        # calculate projection matrix
        proj = np.hstack(
            [pairs[0][1].reshape(dim, 1), pairs[1][1].reshape(dim, 1)])

        # calculate projection
        parray = np.dot(array, proj)

        return parray

    def plot(self, array):
        """ """

        # test arguments
        if array.shape[1] != 2:
            if self._config['pca']:
                array = self._pca2d(array)
            else: raise TypeError(
                "first argument is required to be an array of shape (n, 2)")

        x, y = array[:, 0], array[:, 1]

        # plot grid
        self._axes.grid(self._config['grid'])

        # plot scattered data
        self._axes.scatter(x, y)

        # (optional) plot title
        self.plot_title()

        return True

class Graph(Plot):

    _config = {
        'padding': (0.1, 0.1, 0.1, 0.1),
        'show_legend': False,
        'legend_fontsize': 9.0,
        'graph_layout': 'layer',
        'graph_direction': 'right',
        'node_style': 'o',
        'edge_width_enabled': True,
        'edge_curvature': 1.0
    }

    def plot(self, G):
        """Plot graph.

        Args:
            G: networkx graph instance
            figure_size (tuple): figure size in inches
                (11.69,8.27) for A4, (16.53,11.69) for A3
            edge_attribute (string): name of edge attribute, that
                determines the edge colors by its sign and the edge width
                by its absolute value.
                default: 'weight'
            edge_color (bool): flag for colored edges
                True: edge colors are determined by the sign of the
                    attribute 'weight'
                False: edges are black
            edge_poscolor (string): name of color for edges with
                positive signed attribute. For a full list of specified
                color names see nemoa.base.nplot.get_color()
            edge_negcolor (string): name of color for edges with
                negative signed attribute. For a full list of specified
                color names see nemoa.base.nplot.get_color()
            edge_curvature (float): value within the intervall [-1, 1],
                that determines the curvature of the edges.
                Thereby 1 equals max convexity and -1 max concavity.
            direction (string): string within the list ['up', 'down',
                'left', 'right'], that dermines the plot direction of the
                graph. 'up' means, the first layer is at the bottom.
            edge_style (string):  '-', '<-', '<->', '->',
                '<|-', '<|-|>', '-|>', '|-', '|-|', '-|',
                ']-', ']-[', '-[', 'fancy', 'simple', 'wedge'

        Returns:
            Boolen value which is True if no error occured.

        """
        try:
            import matplotlib.patches
        except ImportError as err:
            raise ImportError(
                "requires package matplotlib: "
                "https://matplotlib.org") from err

        try:
            import networkx as nx
        except ImportError as err:
            raise ImportError(
                "requires package networkx: "
                "https://networkx.github.io") from err

        from nemoa.base import ndict
        from nemoa.math import graph

        # adjust size of subplot
        fig = self._fig
        ax = self._axes
        ax.set_autoscale_on(False)
        figsize = fig.get_size_inches() * fig.dpi
        ax.set_xlim(0., figsize[0])
        ax.set_ylim(0., figsize[1])
        ax.set_aspect('equal', 'box')
        ax.axis('off')

        # get node positions and sizes
        layout_params = ndict.crop(self._config, 'graph_')
        del layout_params['layout']

        pos = graph.get_layout(
            G, layout=self._config['graph_layout'], size=figsize,
            padding=self._config['padding'], **layout_params)

        sizes = graph.get_layout_normsize(pos)
        node_size = sizes.get('node_size', None)
        node_radius = sizes.get('node_radius', None)
        line_width = sizes.get('line_width', None)
        edge_width = sizes.get('edge_width', None)
        font_size = sizes.get('font_size', None)

        # get nodes and groups sorted by node attribute group_id
        groups = graph.get_groups(G, attribute='group')
        sorted_groups = sorted(
            list(groups.keys()),
            key=lambda g: 0 if not isinstance(g, list) or not g \
            else G.node.get(g[0], {}).get('group_id', 0))

        # draw nodes, labeled by groups
        for group in sorted_groups:
            gnodes = groups.get(group, [])
            if not gnodes:
                continue
            refnode = G.node.get(gnodes[0])
            label = refnode['description'] or refnode['group'] or str(group)

            # draw nodes in group
            node_obj = nx.draw_networkx_nodes(
                G, pos, nodelist=gnodes, linewidths=line_width,
                node_size=node_size, node_shape=self._config['node_style'],
                node_color=get_color(refnode['color'], 'white'), label=label)
            node_obj.set_edgecolor(
                get_color(refnode['border_color'], 'black'))

        # draw node labels
        for node, data in G.nodes(data=True):

            # determine label, fontsize and color
            node_label = data.get('label', str(node).title())
            node_label_format = get_texlabel(node_label)
            node_label_size = np.sqrt(get_texlabel_width(node_label))
            font_color = get_color(data['font_color'], 'black')

            # draw node label
            nx.draw_networkx_labels(
                G, pos, labels={node: node_label_format},
                font_size=font_size / node_label_size, font_color=font_color,
                font_family='sans-serif', font_weight='normal')

            # patch node for edges
            circle = matplotlib.patches.Circle(
                pos.get(node), alpha=0., radius=node_radius)
            ax.add_patch(circle)
            G.node[node]['patch'] = circle

        # draw edges
        seen = {}
        if graph.is_directed(G):
            default_edge_style = '-|>'
        else: default_edge_style = '-'

        for (u, v, data) in G.edges(data=True):
            weight = data['weight']
            if weight == 0.:
                continue

            # calculate edge curvature from node positions
            # parameter rad describes the height in the normalized triangle
            if (u, v) in seen:
                rad = seen.get((u, v))
                rad = -(rad + float(np.sign(rad)) * .2)
            else:
                scale = 1. / np.amax(np.array(figsize))
                vec = scale * (np.array(pos[v]) - np.array(pos[u]))
                rad = vec[0] * vec[1] / np.sqrt(2 * np.sum(vec ** 2))
                if self._config['graph_layout'] == 'layer':
                    gdir = self._config['graph_direction']
                    if gdir in ['left', 'right']:
                        rad *= -1
            seen[(u, v)] = rad

            # determine style of edge from edge weight
            if weight is None:
                linestyle = '-'
                linewidth = 0.5 * edge_width
                alpha = 0.5
            elif not self._config['edge_width_enabled']:
                linestyle = '-'
                linewidth = edge_width
                alpha = np.amin([np.absolute(weight), 1.0])
            else:
                linestyle = '-'
                linewidth = np.absolute(weight) * edge_width
                alpha = np.amin([np.absolute(weight), 1.0])

            # draw edge
            node_a = G.node[u]['patch']
            node_b = G.node[v]['patch']
            arrow = matplotlib.patches.FancyArrowPatch(
                posA=node_a.center, posB=node_b.center,
                patchA=node_a, patchB=node_b,
                arrowstyle=default_edge_style,
                connectionstyle='arc3,rad=%s' % rad,
                mutation_scale=linewidth * 12.,
                linewidth=linewidth, linestyle=linestyle,
                color=get_color(data.get('color', 'black')), alpha=alpha)
            ax.add_patch(arrow)

        # (optional) draw legend
        if self._config['show_legend']:
            num_groups = np.sum([1 for g in list(groups.values()) \
                if isinstance(g, list) and g])
            markerscale = 0.6 * self._config['legend_fontsize'] / font_size
            ax.legend(
                numpoints=1,
                loc='lower center',
                ncol=num_groups,
                borderaxespad=0.,
                framealpha=0.,
                bbox_to_anchor=(0.5, -0.1),
                fontsize=self._config['legend_fontsize'],
                markerscale=markerscale)

        # (optional) plot title
        self.plot_title()

        return True

def get_color(*args):
    """Convert color name of XKCD color name survey to RGBA tuple.

    Args:
        List of color names. If the list is empty, a full list of
        available color names is returned. Otherwise the first valid
        color in the list is returned as RGBA tuple. If no color is
        valid None is returned.

    """
    try:
        from matplotlib import colors
    except ImportError as err:
        raise ImportError(
            "requires package matplotlib: "
            "https://matplotlib.org") from err

    if not args:
        clist = list(colors.get_named_colors_mapping().keys())
        return sorted([cname[5:].title() \
            for cname in clist if cname[:5] == 'xkcd:'])

    rgb = None
    for cname in args:
        try:
            rgb = colors.to_rgb('xkcd:%s' % cname)
            break
        except ValueError:
            continue
    return rgb

def get_texlabel(string):
    """Return formated node label as used for plots."""
    lstr = string.rstrip('1234567890')
    if len(lstr) == len(string):
        return '${%s}$' % (string)
    rnum = int(string[len(lstr):])
    lstr = lstr.strip('_')
    return '${%s}_{%i}$' % (lstr, rnum)

def get_texlabel_width(string):
    """Return estimated width for formated node labels."""
    lstr = string.rstrip('1234567890')
    if len(lstr) == len(string):
        return len(string)
    lstr = lstr.strip('_')
    rstr = str(int(string[len(lstr):]))
    return len(lstr) + 0.7 * len(rstr)

def filetypes():
    """Return supported image filetypes."""
    try:
        import matplotlib.pyplot as plt
    except ImportError as err:
        raise ImportError(
            "requires package matplotlib: "
            "https://matplotlib.org") from err

    return plt.gcf().canvas.get_supported_filetypes()


'''
Compare the regridded versions of the SD datasets.
'''

from spectral_cube import SpectralCube
import matplotlib.pyplot as plt
import os
from corner import hist2d
from radio_beam import Beam
import astropy.units as u
import numpy as np

from paths import fourteenB_HI_data_path, data_path
from galaxy_params import gal

# Load in the 4 cubes and run.

vla_cube = SpectralCube.read(fourteenB_HI_data_path("M33_14B-088_HI.clean.image.fits"))

arecibo_path = os.path.join(data_path, "Arecibo")

# Spectral interpolation, followed by reprojection.
arecibo_name = \
    os.path.join(arecibo_path,
                 "14B-088_items_new/m33_arecibo_14B088.fits")
arecibo_cube = SpectralCube.read(arecibo_name)

ebhis_path = os.path.join(data_path, "EBHIS")

# Spectral interpolation, followed by reprojection.
ebhis_name = os.path.join(ebhis_path, "14B-088_items/m33_ebhis_14B088.fits")
ebhis_cube = SpectralCube.read(ebhis_name)

gbt_path = os.path.join(data_path, "GBT")
gbt_name = os.path.join(gbt_path, "14B-088_items/m33_gbt_vlsr_highres_Tmb_14B088.fits")
gbt_cube = SpectralCube.read(gbt_name)

gbt_lowres_name = os.path.join(gbt_path, "14B-088_items/m33_gbt_vlsr_Tmb_14B088.fits")
gbt_lowres_cube = SpectralCube.read(gbt_lowres_name)

# Compare total emission in the cubes.
vla_mask = np.isfinite(vla_cube[0])

arecibo_sum = arecibo_cube.with_mask(vla_mask).sum()
ebhis_sum = ebhis_cube.with_mask(vla_mask).sum()
gbt_sum = gbt_cube.with_mask(vla_mask).sum()
gbt_lowres_sum = gbt_lowres_cube.with_mask(vla_mask).sum()

plt.plot(arecibo_sum, ebhis_sum, gbt_sum, gbt_lowres_sum)

# Compare intensities in one plane

# arecibo_plane = arecibo_cube[500]
# ebhis_plane = ebhis_cube[500]
# gbt_plane = gbt_cube[500]
# gbt_plane[np.isnan(gbt_plane)] = 0.0 * u.K
# gbt_lowres_plane = gbt_lowres_cube[500]

# # Convolve GBT to match EBHIS
# beam_fwhm = lambda diam: ((1.2 * 21 * u.cm) / diam.to(u.cm)) * u.rad
# gbt_90m_beam = Beam(beam_fwhm(90 * u.m))

# gbt_plane._beam = gbt_90m_beam

# gbt_plane_convolved = gbt_plane.convolve_to(ebhis_plane.beam)

# gbt_100m_beam = Beam(beam_fwhm(100 * u.m))
# gbt_plane._beam = gbt_100m_beam

# gbt_plane_convolved_100 = gbt_plane.convolve_to(ebhis_plane.beam)

# ax = plt.subplot(131)
# hist2d(gbt_plane.value.ravel(), ebhis_plane.value.ravel(), ax=ax)
# plt.plot([0, 15], [0, 15])
# ax2 = plt.subplot(132)
# hist2d(gbt_plane_convolved.value.ravel(), ebhis_plane.value.ravel(), ax=ax2)
# plt.plot([0, 15], [0, 15])
# ax3 = plt.subplot(133)
# hist2d(gbt_plane_convolved_100.value.ravel(), ebhis_plane.value.ravel(), ax=ax3)
# plt.plot([0, 15], [0, 15])

# Best match for GBT is with a 106 m beam, convolved to the 80 m of EBHIS.
# Well, something is wrong here. It has to be that the difference between the
# data is a 80 m deconvolved w/ a 106 m beam. The EBHIS beam size should then
# be slightly smaller?

# Now convolve the Arecibo down to the GBT.
# gbt_90m_beam = Beam(beam_fwhm(90 * u.m))

# arecibo_plane_convolved = arecibo_plane.convolve_to(gbt_90m_beam)

# gbt_100m_beam = Beam(beam_fwhm(100 * u.m))

# arecibo_plane_convolved_100 = arecibo_plane.convolve_to(gbt_100m_beam)

# ax = plt.subplot(131)
# hist2d(arecibo_plane.value.ravel(), gbt_plane.value.ravel(), ax=ax)
# plt.plot([0, 15], [0, 15])
# ax2 = plt.subplot(132)
# hist2d(arecibo_plane_convolved.value.ravel(), gbt_plane.value.ravel(), ax=ax2)
# plt.plot([0, 15], [0, 15])
# ax3 = plt.subplot(133)
# hist2d(arecibo_plane_convolved_100.value.ravel(), gbt_plane.value.ravel(), ax=ax3)
# plt.plot([0, 15], [0, 15])


"""
printing tools
"""

import sys
from pandas.core.dtypes.inference import is_sequence
from pandas import compat
from pandas.compat import u
from pandas.core.config import get_option

def adjoin(space, *lists, **kwargs):
    """
    Glues together two sets of strings using the amount of space requested.
    The idea is to prettify.

    ----------
    space : int
        number of spaces for padding
    lists : str
        list of str which being joined
    strlen : callable
        function used to calculate the length of each str. Needed for unicode
        handling.
    justfunc : callable
        function used to justify str. Needed for unicode handling.
    """
    strlen = kwargs.pop('strlen', len)
    justfunc = kwargs.pop('justfunc', justify)

    out_lines = []
    newLists = []
    lengths = [max(map(strlen, x)) + space for x in lists[:-1]]
    # not the last one
    lengths.append(max(map(len, lists[-1])))
    maxLen = max(map(len, lists))
    for i, lst in enumerate(lists):
        nl = justfunc(lst, lengths[i], mode='left')
        nl.extend([' ' * lengths[i]] * (maxLen - len(lst)))
        newLists.append(nl)
    toJoin = zip(*newLists)
    for lines in toJoin:
        out_lines.append(_join_unicode(lines))
    return _join_unicode(out_lines, sep='\n')

def justify(texts, max_len, mode='right'):
    """
    Perform ljust, center, rjust against string or list-like
    """
    if mode == 'left':
        return [x.ljust(max_len) for x in texts]
    elif mode == 'center':
        return [x.center(max_len) for x in texts]
    else:
        return [x.rjust(max_len) for x in texts]

def _join_unicode(lines, sep=''):
    try:
        return sep.join(lines)
    except UnicodeDecodeError:
        sep = compat.text_type(sep)
        return sep.join([x.decode('utf-8') if isinstance(x, str) else x
                         for x in lines])

# Unicode consolidation
# ---------------------
#
# pprinting utility functions for generating Unicode text or
# bytes(3.x)/str(2.x) representations of objects.
# Try to use these as much as possible rather then rolling your own.
#
# When to use
# -----------
#
# 1) If you're writing code internal to pandas (no I/O directly involved),
#    use pprint_thing().
#
#    It will always return unicode text which can handled by other
#    parts of the package without breakage.
#
# 2) If you need to send something to the console, use console_encode().
#
#    console_encode() should (hopefully) choose the right encoding for you
#    based on the encoding set in option "display.encoding"
#
# 3) if you need to write something out to file, use
#    pprint_thing_encoded(encoding).
#
#    If no encoding is specified, it defaults to utf-8. Since encoding pure
#    ascii with utf-8 is a no-op you can safely use the default utf-8 if you're
#    working with straight ascii.

def _pprint_seq(seq, _nest_lvl=0, max_seq_items=None, **kwds):
    """
    internal. pprinter for iterables. you should probably use pprint_thing()
    rather then calling this directly.

    bounds length of printed sequence, depending on options
    """
    if isinstance(seq, set):
        fmt = u("{{{body}}}")
    else:
        fmt = u("[{body}]") if hasattr(seq, '__setitem__') else u("({body})")

    if max_seq_items is False:
        nitems = len(seq)
    else:
        nitems = max_seq_items or get_option("max_seq_items") or len(seq)

    s = iter(seq)
    r = []
    for i in range(min(nitems, len(seq))):  # handle sets, no slicing
        r.append(pprint_thing(
            next(s), _nest_lvl + 1, max_seq_items=max_seq_items, **kwds))
    body = ", ".join(r)

    if nitems < len(seq):
        body += ", ..."
    elif isinstance(seq, tuple) and len(seq) == 1:
        body += ','

    return fmt.format(body=body)

def _pprint_dict(seq, _nest_lvl=0, max_seq_items=None, **kwds):
    """
    internal. pprinter for iterables. you should probably use pprint_thing()
    rather then calling this directly.
    """
    fmt = u("{{{things}}}")
    pairs = []

    pfmt = u("{key}: {val}")

    if max_seq_items is False:
        nitems = len(seq)
    else:
        nitems = max_seq_items or get_option("max_seq_items") or len(seq)

    for k, v in list(seq.items())[:nitems]:
        pairs.append(
            pfmt.format(
                key=pprint_thing(k, _nest_lvl + 1,
                                 max_seq_items=max_seq_items, **kwds),
                val=pprint_thing(v, _nest_lvl + 1,
                                 max_seq_items=max_seq_items, **kwds)))

    if nitems < len(seq):
        return fmt.format(things=", ".join(pairs) + ", ...")
    else:
        return fmt.format(things=", ".join(pairs))

def pprint_thing(thing, _nest_lvl=0, escape_chars=None, default_escapes=False,
                 quote_strings=False, max_seq_items=None):
    """
    This function is the sanctioned way of converting objects
    to a unicode representation.

    properly handles nested sequences containing unicode strings
    (unicode(object) does not)

    Parameters
    ----------
    thing : anything to be formatted
    _nest_lvl : internal use only. pprint_thing() is mutually-recursive
        with pprint_sequence, this argument is used to keep track of the
        current nesting level, and limit it.
    escape_chars : list or dict, optional
        Characters to escape. If a dict is passed the values are the
        replacements
    default_escapes : bool, default False
        Whether the input escape characters replaces or adds to the defaults
    max_seq_items : False, int, default None
        Pass thru to other pretty printers to limit sequence printing

    Returns
    -------
    result - unicode object on py2, str on py3. Always Unicode.

    """

    def as_escaped_unicode(thing, escape_chars=escape_chars):
        # Unicode is fine, else we try to decode using utf-8 and 'replace'
        # if that's not it either, we have no way of knowing and the user
        # should deal with it himself.

        try:
            result = compat.text_type(thing)  # we should try this first
        except UnicodeDecodeError:
            # either utf-8 or we replace errors
            result = str(thing).decode('utf-8', "replace")

        translate = {'\t': r'\t', '\n': r'\n', '\r': r'\r', }
        if isinstance(escape_chars, dict):
            if default_escapes:
                translate.update(escape_chars)
            else:
                translate = escape_chars
            escape_chars = list(escape_chars.keys())
        else:
            escape_chars = escape_chars or tuple()
        for c in escape_chars:
            result = result.replace(c, translate[c])

        return compat.text_type(result)

    if (compat.PY3 and hasattr(thing, '__next__')) or hasattr(thing, 'next'):
        return compat.text_type(thing)
    elif (isinstance(thing, dict) and
          _nest_lvl < get_option("display.pprint_nest_depth")):
        result = _pprint_dict(thing, _nest_lvl, quote_strings=True,
                              max_seq_items=max_seq_items)
    elif (is_sequence(thing) and
          _nest_lvl < get_option("display.pprint_nest_depth")):
        result = _pprint_seq(thing, _nest_lvl, escape_chars=escape_chars,
                             quote_strings=quote_strings,
                             max_seq_items=max_seq_items)
    elif isinstance(thing, compat.string_types) and quote_strings:
        if compat.PY3:
            fmt = u("'{thing}'")
        else:
            fmt = u("u'{thing}'")
        result = fmt.format(thing=as_escaped_unicode(thing))
    else:
        result = as_escaped_unicode(thing)

    return compat.text_type(result)  # always unicode

def pprint_thing_encoded(object, encoding='utf-8', errors='replace', **kwds):
    value = pprint_thing(object)  # get unicode representation of object
    return value.encode(encoding, errors, **kwds)

def _enable_data_resource_formatter(enable):
    if 'IPython' not in sys.modules:
        # definitely not in IPython
        return
    from IPython import get_ipython
    ip = get_ipython()
    if ip is None:
        # still not in IPython
        return

    formatters = ip.display_formatter.formatters
    mimetype = "application/vnd.dataresource+json"

    if enable:
        if mimetype not in formatters:
            # define tableschema formatter
            from IPython.core.formatters import BaseFormatter

            class TableSchemaFormatter(BaseFormatter):
                print_method = '_repr_data_resource_'
                _return_type = (dict,)
            # register it:
            formatters[mimetype] = TableSchemaFormatter()
        # enable it if it's been disabled:
        formatters[mimetype].enabled = True
    else:
        # unregister tableschema mime-type
        if mimetype in formatters:
            formatters[mimetype].enabled = False