#Provides interface functions to create and save models

import numpy
import re
import nltk
import sys
from sklearn.feature_extraction.text import CountVectorizer
import pickle
import os
import sklearn.ensemble
from itertools import chain

base_path = os.path.dirname(__file__)
sys.path.append(base_path)

from .essay_set import EssaySet
from . import util_functions
from . import feature_extractor
import logging
from . import predictor_extractor

log=logging.getLogger()

def read_in_test_data(filename):
    """
    Reads in test data file found at filename.
    filename must be a tab delimited file with columns id, dummy number column, score, dummy score, text
    returns the score and the text
    """
    tid, e_set, score, score2, text = [], [], [], [], []
    combined_raw = open(filename).read()
    raw_lines = combined_raw.splitlines()
    for row in range(1, len(raw_lines)):
        tid1, set1, score1, score12, text1 = raw_lines[row].strip().split("\t")
        tid.append(int(tid1))
        text.append(text1)
        e_set.append(int(set1))
        score.append(int(score1))
        score2.append(int(score12))

    return score, text

def read_in_test_prompt(filename):
    """
    Reads in the prompt from a text file
    Returns string
    """
    prompt_string = open(filename).read()
    return prompt_string

def read_in_test_data_twocolumn(filename,sep=","):
    """
    Reads in a two column version of the test data.
    Filename must point to a delimited file.
    In filename, the first column should be integer score data.
    The second column should be string text data.
    Sep specifies the type of separator between fields.
    """
    score, text = [], []
    combined_raw = open(filename).read()
    raw_lines = combined_raw.splitlines()
    for row in range(1, len(raw_lines)):
        score1, text1 = raw_lines[row].strip().split("\t")
        text.append(text1)
        score.append(int(score1))

    return score, text

def create_essay_set(text, score, prompt_string, generate_additional=True):
    """
    Creates an essay set from given data.
    Text should be a list of strings corresponding to essay text.
    Score should be a list of scores where score[n] corresponds to text[n]
    Prompt string is just a string containing the essay prompt.
    Generate_additional indicates whether to generate additional essays at the minimum score point or not.
    """
    x = EssaySet()
    for i in range(0, len(text)):
        x.add_essay(text[i], score[i])
        if score[i] == min(score) and generate_additional == True:
            x.generate_additional_essays(x._clean_text[len(x._clean_text) - 1], score[i])

    x.update_prompt(prompt_string)

    return x

def get_cv_error(clf,feats,scores):
    """
    Gets cross validated error for a given classifier, set of features, and scores
    clf - classifier
    feats - features to feed into the classified and cross validate over
    scores - scores associated with the features -- feature row 1 associates with score 1, etc.
    """
    results={'success' : False, 'kappa' : 0, 'mae' : 0}
    try:
        cv_preds=util_functions.gen_cv_preds(clf,feats,scores)
        err=numpy.mean(numpy.abs(numpy.array(cv_preds)-scores))
        kappa=util_functions.quadratic_weighted_kappa(list(cv_preds),scores)
        results['mae']=err
        results['kappa']=kappa
        results['success']=True
    except ValueError as ex:
        # If this is hit, everything is fine.  It is hard to explain why the error occurs, but it isn't a big deal.
        msg = u"Not enough classes (0,1,etc) in each cross validation fold: {ex}".format(ex=ex)
        log.debug(msg)
    except:
        log.exception("Error getting cv error estimates.")

    return results

def get_algorithms(algorithm):
    """
    Gets two classifiers for each type of algorithm, and returns them.  First for predicting, second for cv error.
    type - one of util_functions.AlgorithmTypes
    """
    if algorithm == util_functions.AlgorithmTypes.classification:
        clf = sklearn.ensemble.GradientBoostingClassifier(n_estimators=100, learning_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
        clf2=sklearn.ensemble.GradientBoostingClassifier(n_estimators=100, learning_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
    else:
        clf = sklearn.ensemble.GradientBoostingRegressor(n_estimators=100, learning_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
        clf2=sklearn.ensemble.GradientBoostingRegressor(n_estimators=100, learning_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
    return clf, clf2

def extract_features_and_generate_model_predictors(predictor_set, algorithm=util_functions.AlgorithmTypes.regression):
    """
    Extracts features and generates predictors based on a given predictor set
    predictor_set - a PredictorSet object that has been initialized with data
    type - one of util_functions.AlgorithmType
    """
    if(algorithm not in [util_functions.AlgorithmTypes.regression, util_functions.AlgorithmTypes.classification]):
        algorithm = util_functions.AlgorithmTypes.regression

    f = predictor_extractor.PredictorExtractor()
    f.initialize_dictionaries(predictor_set)

    train_feats = f.gen_feats(predictor_set)

    clf,clf2 = get_algorithms(algorithm)
    cv_error_results=get_cv_error(clf2,train_feats,predictor_set._target)

    try:
        set_score = numpy.asarray(predictor_set._target, dtype=numpy.int)
        clf.fit(train_feats, set_score)
    except ValueError:
        log.exception("Not enough classes (0,1,etc) in sample.")
        set_score = predictor_set._target
        set_score[0]=1
        set_score[1]=0
        clf.fit(train_feats, set_score)

    return f, clf, cv_error_results

def extract_features_and_generate_model(essays, algorithm=util_functions.AlgorithmTypes.regression):
    """
    Feed in an essay set to get feature vector and classifier
    essays must be an essay set object
    additional array is an optional argument that can specify
    a numpy array of values to add in
    returns a trained FeatureExtractor object and a trained classifier
    """
    f = feature_extractor.FeatureExtractor()
    f.initialize_dictionaries(essays)

    train_feats = f.gen_feats(essays)

    set_score = numpy.asarray(essays._score, dtype=numpy.int)
    if len(util_functions.f7(list(set_score)))>5:
        algorithm = util_functions.AlgorithmTypes.regression
    else:
        algorithm = util_functions.AlgorithmTypes.classification

    clf,clf2 = get_algorithms(algorithm)

    cv_error_results=get_cv_error(clf2,train_feats,essays._score)

    try:
        clf.fit(train_feats, set_score)
    except ValueError:
        log.exception("Not enough classes (0,1,etc) in sample.")
        set_score[0]=1
        set_score[1]=0
        clf.fit(train_feats, set_score)

    return f, clf, cv_error_results

def dump_model_to_file(prompt_string, feature_ext, classifier, text, score, model_path):
    """
    Writes out a model to a file.
    prompt string is a string containing the prompt
    feature_ext is a trained FeatureExtractor object
    classifier is a trained classifier
    model_path is the path of write out the model file to
    """
    model_file = {'prompt': prompt_string, 'extractor': feature_ext, 'model': classifier, 'text' : text, 'score' : score}
    pickle.dump(model_file, file=open(model_path, "w"))

def create_essay_set_and_dump_model(text,score,prompt,model_path,additional_array=None):
    """
    Function that creates essay set, extracts features, and writes out model
    See above functions for argument descriptions
    """
    essay_set=create_essay_set(text,score,prompt)
    feature_ext,clf=extract_features_and_generate_model(essay_set,additional_array)
    dump_model_to_file(prompt,feature_ext,clf,model_path)


import numpy as np
import os.path as op
from numpy.testing import assert_array_almost_equal, assert_raises
from nose.tools import assert_true

from mne import pick_types, Epochs, read_events
from mne.io import RawArray, read_raw_fif
from mne.utils import requires_version, slow_test, run_tests_if_main
from mne.time_frequency import psd_welch, psd_multitaper

base_dir = op.join(op.dirname(__file__), '..', '..', 'io', 'tests', 'data')
raw_fname = op.join(base_dir, 'test_raw.fif')
event_fname = op.join(base_dir, 'test-eve.fif')

@requires_version('scipy', '0.12')
def test_psd():
    """Tests the welch and multitaper PSD."""
    raw = read_raw_fif(raw_fname)
    picks_psd = [0, 1]

    # Populate raw with sinusoids
    rng = np.random.RandomState(40)
    data = 0.1 * rng.randn(len(raw.ch_names), raw.n_times)
    freqs_sig = [8., 50.]
    for ix, freq in zip(picks_psd, freqs_sig):
        data[ix, :] += 2 * np.sin(np.pi * 2. * freq * raw.times)
    first_samp = raw._first_samps[0]
    raw = RawArray(data, raw.info)

    tmin, tmax = 0, 20  # use a few seconds of data
    fmin, fmax = 2, 70  # look at frequencies between 2 and 70Hz
    n_fft = 128

    # -- Raw --
    kws_psd = dict(tmin=tmin, tmax=tmax, fmin=fmin, fmax=fmax,
                   picks=picks_psd)  # Common to all
    kws_welch = dict(n_fft=n_fft)
    kws_mt = dict(low_bias=True)
    funcs = [(psd_welch, kws_welch),
             (psd_multitaper, kws_mt)]

    for func, kws in funcs:
        kws = kws.copy()
        kws.update(kws_psd)
        psds, freqs = func(raw, proj=False, **kws)
        psds_proj, freqs_proj = func(raw, proj=True, **kws)

        assert_true(psds.shape == (len(kws['picks']), len(freqs)))
        assert_true(np.sum(freqs < 0) == 0)
        assert_true(np.sum(psds < 0) == 0)

        # Is power found where it should be
        ixs_max = np.argmax(psds, axis=1)
        for ixmax, ifreq in zip(ixs_max, freqs_sig):
            # Find nearest frequency to the "true" freq
            ixtrue = np.argmin(np.abs(ifreq - freqs))
            assert_true(np.abs(ixmax - ixtrue) < 2)

        # Make sure the projection doesn't change channels it shouldn't
        assert_array_almost_equal(psds, psds_proj)
        # Array input shouldn't work
        assert_raises(ValueError, func, raw[:3, :20][0])

    # test n_per_seg in psd_welch (and padding)
    psds1, freqs1 = psd_welch(raw, proj=False, n_fft=128, n_per_seg=128,
                              **kws_psd)
    psds2, freqs2 = psd_welch(raw, proj=False, n_fft=256, n_per_seg=128,
                              **kws_psd)
    assert_true(len(freqs1) == np.floor(len(freqs2) / 2.))
    assert_true(psds1.shape[-1] == np.floor(psds2.shape[-1] / 2.))

    # tests ValueError when n_per_seg=None and n_fft > signal length
    kws_psd.update(dict(n_fft=tmax * 1.1 * raw.info['sfreq']))
    assert_raises(ValueError, psd_welch, raw, proj=False, n_per_seg=None,
                  **kws_psd)
    # ValueError when n_overlap > n_per_seg
    kws_psd.update(dict(n_fft=128, n_per_seg=64, n_overlap=90))
    assert_raises(ValueError, psd_welch, raw, proj=False, **kws_psd)

    # -- Epochs/Evoked --
    events = read_events(event_fname)
    events[:, 0] -= first_samp
    tmin, tmax, event_id = -0.5, 0.5, 1
    epochs = Epochs(raw, events[:10], event_id, tmin, tmax, picks=picks_psd,
                    proj=False, preload=True, baseline=None)
    evoked = epochs.average()

    tmin_full, tmax_full = -1, 1
    epochs_full = Epochs(raw, events[:10], event_id, tmin_full, tmax_full,
                         picks=picks_psd, proj=False, preload=True,
                         baseline=None)
    kws_psd = dict(tmin=tmin, tmax=tmax, fmin=fmin, fmax=fmax,
                   picks=picks_psd)  # Common to all
    funcs = [(psd_welch, kws_welch),
             (psd_multitaper, kws_mt)]

    for func, kws in funcs:
        kws = kws.copy()
        kws.update(kws_psd)

        psds, freqs = func(
            epochs[:1], proj=False, **kws)
        psds_proj, freqs_proj = func(
            epochs[:1], proj=True, **kws)
        psds_f, freqs_f = func(
            epochs_full[:1], proj=False, **kws)

        # this one will fail if you add for example 0.1 to tmin
        assert_array_almost_equal(psds, psds_f, 27)
        # Make sure the projection doesn't change channels it shouldn't
        assert_array_almost_equal(psds, psds_proj, 27)

        # Is power found where it should be
        ixs_max = np.argmax(psds.mean(0), axis=1)
        for ixmax, ifreq in zip(ixs_max, freqs_sig):
            # Find nearest frequency to the "true" freq
            ixtrue = np.argmin(np.abs(ifreq - freqs))
            assert_true(np.abs(ixmax - ixtrue) < 2)
        assert_true(psds.shape == (1, len(kws['picks']), len(freqs)))
        assert_true(np.sum(freqs < 0) == 0)
        assert_true(np.sum(psds < 0) == 0)

        # Array input shouldn't work
        assert_raises(ValueError, func, epochs.get_data())

        # Testing evoked (doesn't work w/ compute_epochs_psd)
        psds_ev, freqs_ev = func(
            evoked, proj=False, **kws)
        psds_ev_proj, freqs_ev_proj = func(
            evoked, proj=True, **kws)

        # Is power found where it should be
        ixs_max = np.argmax(psds_ev, axis=1)
        for ixmax, ifreq in zip(ixs_max, freqs_sig):
            # Find nearest frequency to the "true" freq
            ixtrue = np.argmin(np.abs(ifreq - freqs_ev))
            assert_true(np.abs(ixmax - ixtrue) < 2)

        # Make sure the projection doesn't change channels it shouldn't
        assert_array_almost_equal(psds_ev, psds_ev_proj, 27)
        assert_true(psds_ev.shape == (len(kws['picks']), len(freqs)))

@slow_test
@requires_version('scipy', '0.12')
def test_compares_psd():
    """Test PSD estimation on raw for plt.psd and scipy.signal.welch."""
    raw = read_raw_fif(raw_fname)

    exclude = raw.info['bads'] + ['MEG 2443', 'EEG 053']  # bads + 2 more

    # picks MEG gradiometers
    picks = pick_types(raw.info, meg='grad', eeg=False, stim=False,
                       exclude=exclude)[:2]

    tmin, tmax = 0, 10  # use the first 60s of data
    fmin, fmax = 2, 70  # look at frequencies between 5 and 70Hz
    n_fft = 2048

    # Compute psds with the new implementation using Welch
    psds_welch, freqs_welch = psd_welch(raw, tmin=tmin, tmax=tmax, fmin=fmin,
                                        fmax=fmax, proj=False, picks=picks,
                                        n_fft=n_fft, n_jobs=1)

    # Compute psds with plt.psd
    start, stop = raw.time_as_index([tmin, tmax])
    data, times = raw[picks, start:(stop + 1)]
    from matplotlib.pyplot import psd
    out = [psd(d, Fs=raw.info['sfreq'], NFFT=n_fft) for d in data]
    freqs_mpl = out[0][1]
    psds_mpl = np.array([o[0] for o in out])

    mask = (freqs_mpl >= fmin) & (freqs_mpl <= fmax)
    freqs_mpl = freqs_mpl[mask]
    psds_mpl = psds_mpl[:, mask]

    assert_array_almost_equal(psds_welch, psds_mpl)
    assert_array_almost_equal(freqs_welch, freqs_mpl)

    assert_true(psds_welch.shape == (len(picks), len(freqs_welch)))
    assert_true(psds_mpl.shape == (len(picks), len(freqs_mpl)))

    assert_true(np.sum(freqs_welch < 0) == 0)
    assert_true(np.sum(freqs_mpl < 0) == 0)

    assert_true(np.sum(psds_welch < 0) == 0)
    assert_true(np.sum(psds_mpl < 0) == 0)

run_tests_if_main()


# Authors: Emmanuelle Gouillart <emmanuelle.gouillart@normalesup.org>
#          Gael Varoquaux <gael.varoquaux@normalesup.org>
# License: BSD 3 clause

import numpy as np
import scipy as sp
from scipy import ndimage

from nose.tools import assert_equal, assert_true
from numpy.testing import assert_raises

from sklearn.feature_extraction.image import (
    img_to_graph, grid_to_graph, extract_patches_2d,
    reconstruct_from_patches_2d, PatchExtractor, extract_patches)
from sklearn.utils.graph import connected_components

def test_img_to_graph():
    x, y = np.mgrid[:4, :4] - 10
    grad_x = img_to_graph(x)
    grad_y = img_to_graph(y)
    assert_equal(grad_x.nnz, grad_y.nnz)
    # Negative elements are the diagonal: the elements of the original
    # image. Positive elements are the values of the gradient, they
    # should all be equal on grad_x and grad_y
    np.testing.assert_array_equal(grad_x.data[grad_x.data > 0],
                                  grad_y.data[grad_y.data > 0])

def test_grid_to_graph():
    #Checking that the function works with graphs containing no edges
    size = 2
    roi_size = 1
    # Generating two convex parts with one vertex
    # Thus, edges will be empty in _to_graph
    mask = np.zeros((size, size), dtype=np.bool)
    mask[0:roi_size, 0:roi_size] = True
    mask[-roi_size:, -roi_size:] = True
    mask = mask.reshape(size ** 2)
    A = grid_to_graph(n_x=size, n_y=size, mask=mask, return_as=np.ndarray)
    assert_true(connected_components(A)[0] == 2)

    # Checking that the function works whatever the type of mask is
    mask = np.ones((size, size), dtype=np.int16)
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask)
    assert_true(connected_components(A)[0] == 1)

    # Checking dtype of the graph
    mask = np.ones((size, size))
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask, dtype=np.bool)
    assert_true(A.dtype == np.bool)
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask, dtype=np.int)
    assert_true(A.dtype == np.int)
    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask, dtype=np.float)
    assert_true(A.dtype == np.float)

def test_connect_regions():
    lena = sp.misc.lena()
    for thr in (50, 150):
        mask = lena > thr
        graph = img_to_graph(lena, mask)
        assert_equal(ndimage.label(mask)[1], connected_components(graph)[0])

def test_connect_regions_with_grid():
    lena = sp.misc.lena()
    mask = lena > 50
    graph = grid_to_graph(*lena.shape, mask=mask)
    assert_equal(ndimage.label(mask)[1], connected_components(graph)[0])

    mask = lena > 150
    graph = grid_to_graph(*lena.shape, mask=mask, dtype=None)
    assert_equal(ndimage.label(mask)[1], connected_components(graph)[0])

def _downsampled_lena():
    lena = sp.misc.lena().astype(np.float32)
    lena = (lena[::2, ::2] + lena[1::2, ::2] + lena[::2, 1::2]
            + lena[1::2, 1::2])
    lena = (lena[::2, ::2] + lena[1::2, ::2] + lena[::2, 1::2]
            + lena[1::2, 1::2])
    lena = lena.astype(np.float)
    lena /= 16.0
    return lena

def _orange_lena(lena=None):
    lena = _downsampled_lena() if lena is None else lena
    lena_color = np.zeros(lena.shape + (3,))
    lena_color[:, :, 0] = 256 - lena
    lena_color[:, :, 1] = 256 - lena / 2
    lena_color[:, :, 2] = 256 - lena / 4
    return lena_color

def _make_images(lena=None):
    lena = _downsampled_lena() if lena is None else lena
    # make a collection of lenas
    images = np.zeros((3,) + lena.shape)
    images[0] = lena
    images[1] = lena + 1
    images[2] = lena + 2
    return images

downsampled_lena = _downsampled_lena()
orange_lena = _orange_lena(downsampled_lena)
lena_collection = _make_images(downsampled_lena)

def test_extract_patches_all():
    lena = downsampled_lena
    i_h, i_w = lena.shape
    p_h, p_w = 16, 16
    expected_n_patches = (i_h - p_h + 1) * (i_w - p_w + 1)
    patches = extract_patches_2d(lena, (p_h, p_w))
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w))

def test_extract_patches_all_color():
    lena = orange_lena
    i_h, i_w = lena.shape[:2]
    p_h, p_w = 16, 16
    expected_n_patches = (i_h - p_h + 1) * (i_w - p_w + 1)
    patches = extract_patches_2d(lena, (p_h, p_w))
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w, 3))

def test_extract_patches_all_rect():
    lena = downsampled_lena
    lena = lena[:, 32:97]
    i_h, i_w = lena.shape
    p_h, p_w = 16, 12
    expected_n_patches = (i_h - p_h + 1) * (i_w - p_w + 1)

    patches = extract_patches_2d(lena, (p_h, p_w))
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w))

def test_extract_patches_max_patches():
    lena = downsampled_lena
    i_h, i_w = lena.shape
    p_h, p_w = 16, 16

    patches = extract_patches_2d(lena, (p_h, p_w), max_patches=100)
    assert_equal(patches.shape, (100, p_h, p_w))

    expected_n_patches = int(0.5 * (i_h - p_h + 1) * (i_w - p_w + 1))
    patches = extract_patches_2d(lena, (p_h, p_w), max_patches=0.5)
    assert_equal(patches.shape, (expected_n_patches, p_h, p_w))

    assert_raises(ValueError, extract_patches_2d, lena, (p_h, p_w),
                  max_patches=2.0)
    assert_raises(ValueError, extract_patches_2d, lena, (p_h, p_w),
                  max_patches=-1.0)

def test_reconstruct_patches_perfect():
    lena = downsampled_lena
    p_h, p_w = 16, 16

    patches = extract_patches_2d(lena, (p_h, p_w))
    lena_reconstructed = reconstruct_from_patches_2d(patches, lena.shape)
    np.testing.assert_array_equal(lena, lena_reconstructed)

def test_reconstruct_patches_perfect_color():
    lena = orange_lena
    p_h, p_w = 16, 16

    patches = extract_patches_2d(lena, (p_h, p_w))
    lena_reconstructed = reconstruct_from_patches_2d(patches, lena.shape)
    np.testing.assert_array_equal(lena, lena_reconstructed)

def test_patch_extractor_fit():
    lenas = lena_collection
    extr = PatchExtractor(patch_size=(8, 8), max_patches=100, random_state=0)
    assert_true(extr == extr.fit(lenas))

def test_patch_extractor_max_patches():
    lenas = lena_collection
    i_h, i_w = lenas.shape[1:3]
    p_h, p_w = 8, 8

    max_patches = 100
    expected_n_patches = len(lenas) * max_patches
    extr = PatchExtractor(patch_size=(p_h, p_w), max_patches=max_patches,
                          random_state=0)
    patches = extr.transform(lenas)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w))

    max_patches = 0.5
    expected_n_patches = len(lenas) * int((i_h - p_h + 1) * (i_w - p_w + 1)
                                          * max_patches)
    extr = PatchExtractor(patch_size=(p_h, p_w), max_patches=max_patches,
                          random_state=0)
    patches = extr.transform(lenas)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w))

def test_patch_extractor_max_patches_default():
    lenas = lena_collection
    extr = PatchExtractor(max_patches=100, random_state=0)
    patches = extr.transform(lenas)
    assert_equal(patches.shape, (len(lenas) * 100, 12, 12))

def test_patch_extractor_all_patches():
    lenas = lena_collection
    i_h, i_w = lenas.shape[1:3]
    p_h, p_w = 8, 8
    expected_n_patches = len(lenas) * (i_h - p_h + 1) * (i_w - p_w + 1)
    extr = PatchExtractor(patch_size=(p_h, p_w), random_state=0)
    patches = extr.transform(lenas)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w))

def test_patch_extractor_color():
    lenas = _make_images(orange_lena)
    i_h, i_w = lenas.shape[1:3]
    p_h, p_w = 8, 8
    expected_n_patches = len(lenas) * (i_h - p_h + 1) * (i_w - p_w + 1)
    extr = PatchExtractor(patch_size=(p_h, p_w), random_state=0)
    patches = extr.transform(lenas)
    assert_true(patches.shape == (expected_n_patches, p_h, p_w, 3))

def test_extract_patches_strided():

    image_shapes_1D = [(10,), (10,), (11,), (10,)]
    patch_sizes_1D = [(1,), (2,), (3,), (8,)]
    patch_steps_1D = [(1,), (1,), (4,), (2,)]

    expected_views_1D = [(10,), (9,), (3,), (2,)]
    last_patch_1D = [(10,), (8,), (8,), (2,)]

    image_shapes_2D = [(10, 20), (10, 20), (10, 20), (11, 20)]
    patch_sizes_2D = [(2, 2), (10, 10), (10, 11), (6, 6)]
    patch_steps_2D = [(5, 5), (3, 10), (3, 4), (4, 2)]

    expected_views_2D = [(2, 4), (1, 2), (1, 3), (2, 8)]
    last_patch_2D = [(5, 15), (0, 10), (0, 8), (4, 14)]

    image_shapes_3D = [(5, 4, 3), (3, 3, 3), (7, 8, 9), (7, 8, 9)]
    patch_sizes_3D = [(2, 2, 3), (2, 2, 2), (1, 7, 3), (1, 3, 3)]
    patch_steps_3D = [(1, 2, 10), (1, 1, 1), (2, 1, 3), (3, 3, 4)]

    expected_views_3D = [(4, 2, 1), (2, 2, 2), (4, 2, 3), (3, 2, 2)]
    last_patch_3D = [(3, 2, 0), (1, 1, 1), (6, 1, 6), (6, 3, 4)]

    image_shapes = image_shapes_1D + image_shapes_2D + image_shapes_3D
    patch_sizes = patch_sizes_1D + patch_sizes_2D + patch_sizes_3D
    patch_steps = patch_steps_1D + patch_steps_2D + patch_steps_3D
    expected_views = expected_views_1D + expected_views_2D + expected_views_3D
    last_patches = last_patch_1D + last_patch_2D + last_patch_3D

    for (image_shape, patch_size, patch_step, expected_view,
         last_patch) in zip(image_shapes, patch_sizes, patch_steps,
                            expected_views, last_patches):
        image = np.arange(np.prod(image_shape)).reshape(image_shape)
        patches = extract_patches(image, patch_shape=patch_size,
                                  extraction_step=patch_step)

        ndim = len(image_shape)

        assert_true(patches.shape[:ndim] == expected_view)
        last_patch_slices = [slice(i, i + j, None) for i, j in
                             zip(last_patch, patch_size)]
        assert_true((patches[[slice(-1, None, None)] * ndim] ==
                    image[last_patch_slices].squeeze()).all())

def test_extract_patches_square():
    # test same patch size for all dimensions
    lena = downsampled_lena
    i_h, i_w = lena.shape
    p = 8
    expected_n_patches = ((i_h - p + 1),  (i_w - p + 1))
    patches = extract_patches(lena, patch_shape=p)
    assert_true(patches.shape == (expected_n_patches[0], expected_n_patches[1],
                                  p, p))

def test_width_patch():
    # width and height of the patch should be less than the image
    x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    assert_raises(ValueError, extract_patches_2d, x, (4, 1))
    assert_raises(ValueError, extract_patches_2d, x, (1, 4))


#!/usr/bin/env python
"""
===================================================================
RRTController.py - Rapidly-Exploring Random Trees Motion Controller
===================================================================

Uses Rapidly-exploring Random Tree Algorithm to generate paths given the starting position and the goal point.
"""

from numpy import *
from __is_inside import *
import math
import sys,os
from scipy.linalg import norm
from numpy.matlib import zeros
import __is_inside
import time, sys,os
import scipy as Sci
import scipy.linalg
import Polygon, Polygon.IO
import Polygon.Utils as PolyUtils
import Polygon.Shapes as PolyShapes
from math import sqrt, fabs , pi
import random
import thread
import threading

# importing matplotlib to show the path if possible
try:
    import matplotlib.pyplot as plt
    import matplotlib.animation as animation
    import_matplotlib = True
except:
    print "matplotlib is not imported. Plotting is disabled"
    import_matplotlib = False

class motionControlHandler:
    def __init__(self, proj, shared_data,robot_type,max_angle_goal,max_angle_overlap,plotting):
        """
        Rapidly-Exploring Random Trees alogorithm motion planning controller

        robot_type (int): Which robot is used for execution. BasicSim is 1, ODE is 2, ROS is 3, Nao is 4, Pioneer is 5(default=1)
        max_angle_goal (float): The biggest difference in angle between the new node and the goal point that is acceptable. If it is bigger than the max_angle, the new node will not be connected to the goal point. The value should be within 0 to 6.28 = 2*pi. Default set to 6.28 = 2*pi (default=6.28)
        max_angle_overlap (float): difference in angle allowed for two nodes overlapping each other. If you don't want any node overlapping with each other, put in 2*pi = 6.28. Default set to 1.57 = pi/2 (default=1.57)
        plotting (bool): Check the box to enable plotting. Uncheck to disable plotting (default=True)
        """

        self.system_print       = False       # for debugging. print on GUI ( a bunch of stuffs)
        self.finish_print       = False       # set to 1 to print the original finished E and V before trimming the tree
        self.orientation_print  = False        # show the orientation information of the robot

        # Get references to handlers we'll need to communicate with
        self.drive_handler = proj.h_instance['drive']
        self.pose_handler = proj.h_instance['pose']

        # Get information about regions
        self.proj              = proj
        self.coordmap_map2lab  = proj.coordmap_map2lab
        self.coordmap_lab2map  = proj.coordmap_lab2map
        self.last_warning      = 0
        self.previous_next_reg = None

        # Store the Rapidly-Exploring Random Tress Built
        self.RRT_V              = None               # array containing all the points on the RRT Tree
        self.RRT_E              = None               # array specifying the connection of points on the Tree
        self.E_current_column   = None               # the current column on the tree (to find the current heading point)
        self.Velocity           = None
        self.currentRegionPoly  = None
        self.nextRegionPoly     = None
        self.map                = {}
        self.all                = Polygon.Polygon()
        self.trans_matrix       = mat([[0,1],[-1,0]])   # transformation matrix for find the normal to the vector
        self.stuck_thres        = 20          # threshold for changing the range of sampling omega

        # Information about the robot (default set to ODE)
        if robot_type not in [1,2,3,4,5]:
            robot_type = 1
        self.system = robot_type

        # Information about maximum turning angle allowed from the latest node to the goal point
        if max_angle_goal > 2*pi:
            max_angle_goal = 2*pi
        if max_angle_goal < 0:
            max_angle_goal = 0
        self.max_angle_allowed = max_angle_goal

        # Information about maximum difference in angle allowed between two overlapping nodes
        if max_angle_overlap > 2*pi:
            max_angle_overlap = 2*pi
        if max_angle_overlap < 0:
            max_angle_overlap = 0
        self.max_angle_overlap = max_angle_overlap

        # Information about whether plotting is enabled.
        if plotting is True and import_matplotlib == True:
            self.plotting          = True
        else:
            self.plotting          = False

        # Specify the size of the robot
        # 1: basicSim; 2: ODE; 3: ROS  4: Nao; 5: Pioneer
        #  self.radius: radius of the robot
        #  self.timestep  : number of linear segments to break the curve into for calculation of x, y position
        #  self.step_size  : the length of each step for connection to goal point
        #  self.velocity   : Velocity of the robot in m/s in control space (m/s)
        if self.system  == 1:
            self.radius = 5
            self.step_size  = 25
            self.timeStep = 10
            self.velocity = 2    # 1.5
        if  self.system == 2:
            self.radius = 5
            self.step_size  = 15
            self.timeStep = 10
            self.velocity = 2    # 1.5
        elif self.system == 3:
            self.ROSInitHandler = shared_data['ROS_INIT_HANDLER']
            self.radius = self.ROSInitHandler.robotPhysicalWidth/2
            self.step_size  = self.radius*3 #0.2
            self.timeStep = 8
            self.velocity  = self.radius/2  #0.08
        elif self.system == 4:
            self.radius = 0.15*1.2
            self.step_size  = 0.2      #set the step_size for points be 1/5 of the norm  ORIGINAL = 0.4
            self.timeStep = 5
            self.velocity  = 0.05
        elif self.system == 5:
            self.radius = 0.15
            self.step_size  = 0.2      #set the step_size for points be 1/5 of the norm  ORIGINAL = 0.4
            self.timeStep = 5
            self.velocity  = 0.05

        # Operate_system (int): Which operating system is used for execution.
        # Ubuntu and Mac is 1, Windows is 2
        if sys.platform in ['win32', 'cygwin']:
            self.operate_system = 2
        else:
            self.operate_system = 1

        if self.system_print == True:
            print "The operate_system is "+ str(self.operate_system)

        # Generate polygon for regions in the map
        for region in self.proj.rfi.regions:
            self.map[region.name] = self.createRegionPolygon(region)
            for n in range(len(region.holeList)): # no of holes
                self.map[region.name] -= self.createRegionPolygon(region,n)

        # Generate the boundary polygon
        for regionName,regionPoly in self.map.iteritems():
            self.all += regionPoly

        # Start plotting if operating in Windows
        if self.operate_system == 2 and self.plotting ==True:
            # start using anmination to plot the robot
            self.fig = plt.figure()
            self.ax = self.fig.add_subplot(111)
            self.scope = _Scope(self.ax,self)
            thread.start_new_thread(self.jplot,())

    def gotoRegion(self, current_reg, next_reg, last=False):
        """
        If ``last`` is True, we will move to the center of the destination region.
        Returns ``True`` if we've reached the destination region.
        """

        if current_reg == next_reg and not last:
            # No need to move!
            self.drive_handler.setVelocity(0, 0)  # So let's stop
            return True

        # Find our current configuration
        pose = self.pose_handler.getPose()

        # Check if Vicon has cut out
        # TODO: this should probably go in posehandler?
        if math.isnan(pose[2]):
            print "WARNING: No Vicon data! Pausing."
            self.drive_handler.setVelocity(0, 0)  # So let's stop
            time.sleep(1)
            return False

        ###This part will be run when the robot goes to a new region, otherwise, the original tree will be used.
        if not self.previous_next_reg == next_reg:
            # Entered a new region. New tree should be formed.
            self.nextRegionPoly    = self.map[self.proj.rfi.regions[next_reg].name]
            self.currentRegionPoly = self.map[self.proj.rfi.regions[current_reg].name]

            if self.system_print == True:
                print "next Region is " + str(self.proj.rfi.regions[next_reg].name)
                print "Current Region is " + str(self.proj.rfi.regions[current_reg].name)

            #set to zero velocity before tree is generated
            self.drive_handler.setVelocity(0, 0)
            if last:
                transFace = None
            else:
                # Determine the mid points on the faces connecting to the next region (one goal point will be picked among all the mid points later in buildTree)
                transFace   = None
                q_gBundle   = [[],[]] # list of goal points (midpoints of transition faces)
                face_normal = [[],[]] # normal of the trnasition faces
                for i in range(len(self.proj.rfi.transitions[current_reg][next_reg])):
                    pointArray_transface = [x for x in self.proj.rfi.transitions[current_reg][next_reg][i]]
                    transFace = asarray(map(self.coordmap_map2lab,pointArray_transface))
                    bundle_x = (transFace[0,0] +transFace[1,0])/2    #mid-point coordinate x
                    bundle_y = (transFace[0,1] +transFace[1,1])/2    #mid-point coordinate y
                    q_gBundle     = hstack((q_gBundle,vstack((bundle_x,bundle_y))))

                    #find the normal vector to the face
                    face          = transFace[0,:] - transFace[1,:]
                    distance_face = norm(face)
                    normal        = face/distance_face * self.trans_matrix
                    face_normal   = hstack((face_normal,vstack((normal[0,0],normal[0,1]))))

                if transFace is None:
                    print "ERROR: Unable to find transition face between regions %s and %s.  Please check the decomposition (try viewing projectname_decomposed.regions in RegionEditor or a text editor)." % (self.proj.rfi.regions[current_reg].name, self.proj.rfi.regions[next_reg].name)

            # Run algorithm to build the Rapid-Exploring Random Trees
            self.RRT_V = None
            self.RRT_E = None

            # For plotting
            if self.operate_system == 2:
                if self.plotting == True:
                    self.ax.cla()
                else:
                    self.ax = None
            else:
                self.ax = None

            if self.operate_system == 1 and self.plotting == True:
                plt.cla()
                self.plotMap(self.map)
                plt.plot(pose[0],pose[1],'ko')

            self.RRT_V,self.RRT_E,self.E_current_column = self.buildTree(\
            [pose[0], pose[1]],pose[2],self.currentRegionPoly, self.nextRegionPoly,q_gBundle,face_normal)

            """
            # map the lab coordinates back to pixels
            V_tosend = array(mat(self.RRT_V[1:,:])).T
            V_tosend = map(self.coordmap_lab2map, V_tosend)
            V_tosend = mat(V_tosend).T
            s = 'RRT:E'+"["+str(list(self.RRT_E[0]))+","+str(list(self.RRT_E[1]))+"]"+':V'+"["+str(list(self.RRT_V[0]))+","+str(list(V_tosend[0]))+","+str(list(V_tosend[1]))+"]"+':T'+"["+str(list(q_gBundle[0]))+","+str(list(q_gBundle[1]))+"]"
            #print s
            """

        # Run algorithm to find a velocity vector (global frame) to take the robot to the next region
        self.Velocity = self.getVelocity([pose[0], pose[1]], self.RRT_V,self.RRT_E)
        #self.Node = self.getNode([pose[0], pose[1]], self.RRT_V,self.RRT_E)
        self.previous_next_reg = next_reg

        # Pass this desired velocity on to the drive handler
        self.drive_handler.setVelocity(self.Velocity[0,0], self.Velocity[1,0], pose[2])
        #self.drive_handler.setVelocity(self.Node[0,0], self.Node[1,0], pose[2])
        RobotPoly = Polygon.Shapes.Circle(self.radius,(pose[0],pose[1]))

        # check if robot is inside the current region
        departed = not self.currentRegionPoly.overlaps(RobotPoly)
        arrived  = self.nextRegionPoly.covers(RobotPoly)

        if departed and (not arrived) and (time.time()-self.last_warning) > 0.5:
            # Figure out what region we think we stumbled into
            for r in self.proj.rfi.regions:
                pointArray = [self.coordmap_map2lab(x) for x in r.getPoints()]
                vertices = mat(pointArray).T

                if is_inside([pose[0], pose[1]], vertices):
                    print "I think I'm in " + r.name
                    print pose
                    break
            self.last_warning = time.time()

        #print "arrived:"+str(arrived)
        return arrived

    def createRegionPolygon(self,region,hole = None):
        """
        This function takes in the region points and make it a Polygon.
        """
        if hole == None:
            pointArray = [x for x in region.getPoints()]
        else:
            pointArray = [x for x in region.getPoints(hole_id = hole)]
        pointArray = map(self.coordmap_map2lab, pointArray)
        regionPoints = [(pt[0],pt[1]) for pt in pointArray]
        formedPolygon= Polygon.Polygon(regionPoints)
        return formedPolygon

    def getVelocity(self,p, V, E, last=False):
        """
        This function calculates the velocity for the robot with RRT.
        The inputs are (given in order):
            p        = the current x-y position of the robot
            E        = edges of the tree  (2 x No. of nodes on the tree)
            V        = points of the tree (2 x No. of vertices)
            last = True, if the current region is the last region
                 = False, if the current region is NOT the last region
        """

        pose     = mat(p).T

        #dis_cur = distance between current position and the next point
        dis_cur  = vstack((V[1,E[1,self.E_current_column]],V[2,E[1,self.E_current_column]]))- pose

        heading = E[1,self.E_current_column]        # index of the current heading point on the tree
        if norm(dis_cur) < 1.5*self.radius:         # go to next point
            if not heading == shape(V)[1]-1:
                self.E_current_column = self.E_current_column + 1
                dis_cur  = vstack((V[1,E[1,self.E_current_column]],V[2,E[1,self.E_current_column]]))- pose
            #else:
            #    dis_cur  = vstack((V[1,E[1,self.E_current_column]],V[2,E[1,self.E_current_column]]))- vstack((V[1,E[0,self.E_current_column]],V[2,E[0,self.E_current_column]]))

        Vel = zeros([2,1])
        Vel[0:2,0] = dis_cur/norm(dis_cur)*0.5                    #TUNE THE SPEED LATER
        return Vel

    def getNode(self,p, V, E, last=False):
        """

        This function calculates the velocity for the robot with RRT.
        The inputs are (given in order):
            p        = the current x-y position of the robot

            E        = edges of the tree  (2 x No. of nodes on the tree)
            V        = points of the tree (2 x No. of vertices)
            last = True, if the current region is the last region
                 = False, if the current region is NOT the last region

        """

        pose     = mat(p).T

        #dis_cur = distance between current position and the next point
        dis_cur  = vstack((V[1,E[1,self.E_current_column]],V[2,E[1,self.E_current_column]]))- pose

        heading = E[1,self.E_current_column]        # index of the current heading point on the tree
        if norm(dis_cur) < 1.5*self.radius:         # go to next point
            if not heading == shape(V)[1]-1:
                self.E_current_column = self.E_current_column + 1
                dis_cur  = vstack((V[1,E[1,self.E_current_column]],V[2,E[1,self.E_current_column]]))- pose

        Node = zeros([2,1])
        Node[0,0] = V[1,E[1,self.E_current_column]]
        Node[1,0] = V[2,E[1,self.E_current_column]]
        #Vel[0:2,0] = dis_cur/norm(dis_cur)*0.5                    #TUNE THE SPEED LATER
        return Node

    def buildTree(self,p,theta,regionPoly,nextRegionPoly,q_gBundle,face_normal, last=False):
        """
        This function builds the RRT tree.
        p                : x,y position of the robot
        theta            : current orientation of the robot
        regionPoly       : current region polygon
        nextRegionPoly   : next region polygon
        q_gBundle        : coordinates of q_goals that the robot can reach
        face_normal      : the normal vector of each face corresponding to each goal point in q_gBundle
        """

        q_init          = mat(p).T
        V               = vstack((0,q_init))
        theta           = self.orientation_bound(theta)
        V_theta         = array([theta])

        #!!! CONTROL SPACE: generate a list of omega for random sampling
        omegaLowerBound = -math.pi/20      # upper bound for the value of omega
        omegaUpperBound = math.pi/20       # lower bound for the value of omega
        omegaNoOfSteps  = 20
        self.omega_range = linspace(omegaLowerBound,omegaUpperBound,omegaNoOfSteps)
        self.omega_range_escape = linspace(omegaLowerBound*4,omegaUpperBound*4,omegaNoOfSteps*4)    # range used when stuck > stuck_thres

        regionPolyOld = Polygon.Polygon(regionPoly)
        regionPoly += PolyShapes.Circle(self.radius*2.5,(q_init[0,0],q_init[1,0]))

        # check faces of the current region for goal points
        E     = [[],[]]       # the tree matrix
        Other = [[],[]]
        path                        = False          # if path formed then = 1
        stuck                       = 0              # count for changing the range of sampling omega
        append_after_latest_node    = False       # append new nodes to the latest node

        if self.system_print == True:
            print "plotting in buildTree is " + str(self.plotting)
        if self.plotting == True:
            if not plt.isinteractive():
                plt.ion()
            plt.hold(True)

        while not path:
            #step -1: try connection to q_goal (generate path to goal)
            i = 0

            if self.system_print == True:
                print "Try Connection to the goal points"

            # pushing possible q_goals into the current region (ensure path is covered by the current region polygon)
            q_pass = [[],[],[]]
            q_pass_dist = []
            q_gBundle = mat(q_gBundle)
            face_normal = mat(face_normal)

            while i < q_gBundle.shape[1]:
                q_g_original = q_gBundle[:,i]
                q_g = q_gBundle[:,i]+face_normal[:,i]*1.5*self.radius    ##original 2*self.radius
                #q_g = q_gBundle[:,i]+(q_gBundle[:,i]-V[1:,(shape(V)[1]-1)])/norm(q_gBundle[:,i]-V[1:,(shape(V)[1]-1)])*1.5*self.radius    ##original 2*self.radius
                if not regionPolyOld.isInside(q_g[0],q_g[1]):
                    #q_g = q_gBundle[:,i]-(q_gBundle[:,i]-V[1:,(shape(V)[1]-1)])/norm(q_gBundle[:,i]-V[1:,(shape(V)[1]-1)])*1.5*self.radius    ##original 2*self.radius
                    q_g = q_gBundle[:,i]-face_normal[:,i]*1.5*self.radius    ##original 2*self.radius

                #forming polygon for path checking
                EdgePolyGoal = PolyShapes.Circle(self.radius,(q_g[0,0],q_g[1,0])) + PolyShapes.Circle(self.radius,(V[1,shape(V)[1]-1],V[2:,shape(V)[1]-1]))
                EdgePolyGoal = PolyUtils.convexHull(EdgePolyGoal)
                dist = norm(q_g - V[1:,shape(V)[1]-1])

                #check connection to goal
                connect_goal = regionPoly.covers(EdgePolyGoal)   #check coverage of path from new point to goal

                # compare orientation difference
                thetaPrev = V_theta[shape(V)[1]-1]

                theta_orientation = abs(arctan((q_g[1,0]- V[2,shape(V)[1]-1])/(q_g[0,0]- V[1,shape(V)[1]-1])))
                if q_g[1,0] > V[2,shape(V)[1]-1]:
                    if q_g[0,0] < V[1,shape(V)[1]-1]: # second quadrant
                        theta_orientation = pi - theta_orientation
                    elif q_g[0,0] > V[1,shape(V)[1]-1]: # first quadrant
                        theta_orientation = theta_orientation
                elif q_g[1,0] < V[2,shape(V)[1]-1]:
                    if q_g[0,0] < V[1,shape(V)[1]-1]: #third quadrant
                        theta_orientation = pi + theta_orientation
                    elif q_g[0,0] > V[1,shape(V)[1]-1]: # foruth quadrant
                        theta_orientation =  2*pi - theta_orientation

                # check the angle between vector(new goal to goal_original ) and vector( latest node to new goal)
                Goal_to_GoalOriginal = q_g_original - q_g
                LatestNode_to_Goal   = q_g - V[1:,shape(V)[1]-1]
                Angle_Goal_LatestNode= arccos(vdot(array(Goal_to_GoalOriginal), array(LatestNode_to_Goal))/norm(Goal_to_GoalOriginal)/norm(LatestNode_to_Goal))

                # if connection to goal can be established and the max change in orientation of the robot is smaller than max_angle, tree is said to be completed.
                if self.orientation_print == True:
                    print "theta_orientation is " + str(theta_orientation)
                    print "thetaPrev is " + str(thetaPrev)
                    print "(theta_orientation - thetaPrev) is " + str(abs(theta_orientation - thetaPrev))
                    print "self.max_angle_allowed is " + str(self.max_angle_allowed)
                    print "abs(theta_orientation - thetaPrev) < self.max_angle_allowed" + str(abs(theta_orientation - thetaPrev) < self.max_angle_allowed)
                    print"Goal_to_GoalOriginal: " + str( array(Goal_to_GoalOriginal)) + "; LatestNode_to_Goal: " + str( array(LatestNode_to_Goal))
                    print vdot(array(Goal_to_GoalOriginal), array(LatestNode_to_Goal))
                    print "Angle_Goal_LatestNode" + str(Angle_Goal_LatestNode)

                if connect_goal and (abs(theta_orientation - thetaPrev) < self.max_angle_allowed) and (Angle_Goal_LatestNode < self.max_angle_allowed):

                    path = True
                    q_pass = hstack((q_pass,vstack((i,q_g))))
                    q_pass_dist = hstack((q_pass_dist,dist))

                i = i + 1

            if self.system_print == True:
                print "checked goal points"

            self.E = E
            self.V = V
            # connection to goal has established
            # Obtain the closest goal point that path can be formed.
            if path:
                if shape(q_pass_dist)[0] == 1:
                    cols = 0
                else:
                    (cols,) = nonzero(q_pass_dist == min(q_pass_dist))
                    cols = asarray(cols)[0]
                q_g = q_pass[1:,cols]
                """
                q_g = q_g-(q_gBundle[:,q_pass[0,cols]]-V[1:,(shape(V)[1]-1)])/norm(q_gBundle[:,q_pass[0,cols]]-V[1:,(shape(V)[1]-1)])*3*self.radius   #org 3
                if not nextRegionPoly.isInside(q_g[0],q_g[1]):
                    q_g = q_g+(q_gBundle[:,q_pass[0,cols]]-V[1:,(shape(V)[1]-1)])/norm(q_gBundle[:,q_pass[0,cols]]-V[1:,(shape(V)[1]-1)])*6*self.radius   #org 3
                """
                if self.plotting == True :
                    if self.operate_system == 1:
                        plt.suptitle('Rapidly-exploring Random Tree', fontsize=12)
                        plt.xlabel('x')
                        plt.ylabel('y')
                        if shape(V)[1] <= 2:
                            plt.plot(( V[1,shape(V)[1]-1],q_g[0,0]),( V[2,shape(V)[1]-1],q_g[1,0]),'b')
                        else:
                            plt.plot(( V[1,E[0,shape(E)[1]-1]], V[1,shape(V)[1]-1],q_g[0,0]),( V[2,E[0,shape(E)[1]-1]], V[2,shape(V)[1]-1],q_g[1,0]),'b')
                        plt.plot(q_g[0,0],q_g[1,0],'ko')
                        plt.figure(1).canvas.draw()
                    else:
                        BoundPolyPoints = asarray(PolyUtils.pointList(regionPoly))
                        self.ax.plot(BoundPolyPoints[:,0],BoundPolyPoints[:,1],'k')
                        if shape(V)[1] <= 2:
                            self.ax.plot(( V[1,shape(V)[1]-1],q_g[0,0]),( V[2,shape(V)[1]-1],q_g[1,0]),'b')
                        else:
                            self.ax.plot(( V[1,E[0,shape(E)[1]-1]], V[1,shape(V)[1]-1],q_g[0,0]),( V[2,E[0,shape(E)[1]-1]], V[2,shape(V)[1]-1],q_g[1,0]),'b')
                        self.ax.plot(q_g[0,0],q_g[1,0],'ko')

                # trim the path connecting current node to goal point into pieces if the path is too long now
                numOfPoint = floor(norm(V[1:,shape(V)[1]-1]- q_g)/self.step_size)
                if numOfPoint < 3:
                    numOfPoint = 3
                x = linspace( V[1,shape(V)[1]-1], q_g[0,0], numOfPoint )
                y = linspace( V[2,shape(V)[1]-1], q_g[1,0], numOfPoint )
                for i in range(x.shape[0]):
                    if i != 0:
                        V = hstack((V,vstack((shape(V)[1],x[i],y[i]))))
                        E = hstack((E,vstack((shape(V)[1]-2,shape(V)[1]-1))))

                #push the goal point to the next region
                q_g = q_g+face_normal[:,q_pass[0,cols]]*3*self.radius    ##original 2*self.radius
                if not nextRegionPoly.isInside(q_g[0],q_g[1]):
                    q_g = q_g-face_normal[:,q_pass[0,cols]]*6*self.radius    ##original 2*self.radius
                V = hstack((V,vstack((shape(V)[1],q_g[0,0],q_g[1,0]))))
                E = hstack((E,vstack((shape(V)[1]-2 ,shape(V)[1]-1))))

                if self.plotting == True :
                    if self.operate_system == 1:
                        plt.plot(q_g[0,0],q_g[1,0],'ko')
                        plt.plot(( V[1,shape(V)[1]-1],V[1,shape(V)[1]-2]),( V[2,shape(V)[1]-1],V[2,shape(V)[1]-2]),'b')
                        plt.figure(1).canvas.draw()
                    else:
                        self.ax.plot(q_g[0,0],q_g[1,0],'ko')
                        self.ax.plot(( V[1,shape(V)[1]-1],V[1,shape(V)[1]-2]),( V[2,shape(V)[1]-1],V[2,shape(V)[1]-2]),'b')

            # path is not formed, try to append points onto the tree
            if not path:

                # connection_to_tree : connection to the tree is successful
                if append_after_latest_node:
                    V,V_theta,E,Other,stuck,append_after_latest_node, connection_to_tree = self.generateNewNode(V,V_theta,E,Other,regionPoly,stuck, append_after_latest_node)
                else:
                    connection_to_tree = False
                    while not connection_to_tree:
                        V,V_theta,E,Other,stuck,append_after_latest_node, connection_to_tree = self.generateNewNode  (V,V_theta,E,Other,regionPoly,stuck)

        if self.finish_print:
            print 'Here is the V matrix:', V, 'Here is the E matrix:',E
            print >>sys.__stdout__, 'Here is the V matrix:\n', V, '\nHere is the E matrix:\n',E

        #B: trim to a single path
        single = 0
        while single == 0:
            trim  = 0
            for j in range(shape(V)[1]-3):
                (row,col) = nonzero(E == j+1)
                if len(col) == 1:
                    E = delete(E, col[0], 1)
                    trim = 1

            if trim == 0:
                single = 1;

        ####print with matlib
        if self.plotting ==True :
            if self.operate_system == 1:
                plt.plot(V[1,:],V[2,:],'b')
                for i in range(shape(E)[1]):
                    plt.text(V[1,E[0,i]],V[2,E[0,i]], V[0,E[0,i]], fontsize=12)
                    plt.text(V[1,E[1,i]],V[2,E[1,i]], V[0,E[1,i]], fontsize=12)
                plt.figure(1).canvas.draw()
            else:
                BoundPolyPoints = asarray(PolyUtils.pointList(regionPoly))
                self.ax.plot(BoundPolyPoints[:,0],BoundPolyPoints[:,1],'k')
                self.ax.plot(V[1,:],V[2,:],'b')
                for i in range(shape(E)[1]):
                    self.ax.text(V[1,E[0,i]],V[2,E[0,i]], V[0,E[0,i]], fontsize=12)
                    self.ax.text(V[1,E[1,i]],V[2,E[1,i]], V[0,E[1,i]], fontsize=12)

        #return V, E, and the current node number on the tree
        V = array(V)
        return V, E, 0

    def generateNewNode(self,V,V_theta,E,Other,regionPoly,stuck,append_after_latest_node =False):
        """
        Generate a new node on the current tree matrix
        V         : the node matrix
        V_theta   : the orientation matrix
        E         : the tree matrix (or edge matrix)
        Other     : the matrix containing the velocity and angular velocity(omega) information
        regionPoly: the polygon of current region
        stuck     : count on the number of times failed to generate new node
        append_after_latest_node : append new nodes to the latest node (True only if the previous node addition is successful)
        """

        if self.system_print == True:
            print "In control space generating path,stuck = " + str(stuck)

        connection_to_tree = False   # True when connection to the tree is successful

        if stuck > self.stuck_thres:
            # increase the range of omega since path cannot ge generated
            omega = random.choice(self.omega_range_escape)
        else:
            #!!!! CONTROL SPACE STEP 1 - generate random omega
            omega = random.choice(self.omega_range)

        #!!!! CONTROL SPACE STEP 2 - pick a random point on the tree
        if append_after_latest_node:
            tree_index = shape(V)[1]-1
        else:
            if random.choice([1,2]) == 1:
                tree_index = random.choice(array(V[0])[0])
            else:
                tree_index = shape(V)[1]-1

        xPrev     = V[1,tree_index]
        yPrev     = V[2,tree_index]
        thetaPrev = V_theta[tree_index]

        j = 1
        #!!!! CONTROL SPACE STEP 3 - Check path of the robot
        path_robot = PolyShapes.Circle(self.radius,(xPrev,yPrev))
        while j <= self.timeStep:
            xOrg      = xPrev
            yOrg      = yPrev
            xPrev     = xPrev + self.velocity/omega*(sin(omega* 1 + thetaPrev)-sin(thetaPrev))
            yPrev     = yPrev - self.velocity/omega*(cos(omega* 1 + thetaPrev)-cos(thetaPrev))
            thetaPrev = omega* 1 + thetaPrev
            path_robot = path_robot + PolyShapes.Circle(self.radius,(xPrev,yPrev))

            j = j + 1

        thetaPrev = self.orientation_bound(thetaPrev)
        path_all = PolyUtils.convexHull(path_robot)
        in_bound = regionPoly.covers(path_all)
        """
        # plotting
        if plotting == True:
            self.plotPoly(path_all,'r',1)
        """

        stuck = stuck + 1

        if in_bound:
            robot_new_node = PolyShapes.Circle(self.radius,(xPrev,yPrev))
            # check how many nodes on the tree does the new node overlaps with
            nodes_overlap_count = 0
            for k in range(shape(V)[1]-1):
                robot_old_node = PolyShapes.Circle(self.radius,(V[1,k],V[2,k]))
                if robot_new_node.overlaps(robot_old_node):
                    if  abs(thetaPrev - V_theta[k]) <   self.max_angle_overlap:
                        nodes_overlap_count += 1

            if nodes_overlap_count == 0 or (stuck > self.stuck_thres+1 and nodes_overlap_count < 2) or (stuck > self.stuck_thres+500):
                if  stuck > self.stuck_thres+1:
                    append_after_latest_node  = False

                if (stuck > self.stuck_thres+500):
                    stuck = 0
                stuck = stuck - 20
                # plotting
                if self.plotting == True:
                    self.plotPoly(path_all,'b',1)

                if self.system_print == True:
                    print "node connected"

                V = hstack((V,vstack((shape(V)[1],xPrev,yPrev))))
                V_theta = hstack((V_theta,thetaPrev))
                E = hstack((E,vstack((tree_index ,shape(V)[1]-1))))
                Other = hstack((Other,vstack((self.velocity,omega))))
                ##################### E should add omega and velocity
                connection_to_tree = True
                append_after_latest_node  = True
            else:
                append_after_latest_node = False

                if self.system_print == True:
                    print "node not connected. check goal point"

        else:
            append_after_latest_node = False

        return  V,V_theta,E,Other,stuck,append_after_latest_node, connection_to_tree

    def orientation_bound(self,theta):
        """
        make sure the returned angle is between 0 to 2*pi
        """
        while theta > 2*pi or theta < 0:
            if theta > 2*pi:
                theta = theta - 2*pi
            else:
                theta = theta + 2*pi

        return theta

    def plotMap(self,mappedRegions):
        """
        Plotting regions and obstacles with matplotlib.pyplot

        number: figure number (see on top)
        """

        #if not plt.isinteractive():
        #    plt.ion()
        #plt.hold(True)

        if self.operate_system == 1:
            for regionName,regionPoly in mappedRegions.iteritems():
                self.plotPoly(regionPoly,'k')
            plt.figure(1).canvas.draw()

    def plotPoly(self,c,string,w = 1):
        """
        Plot polygons inside the boundary
        c = polygon to be plotted with matlabplot
        string = string that specify color
        w      = width of the line plotting
        """
        if bool(c):
            for i in range(len(c)):
                #toPlot = Polygon.Polygon(c.contour(i))
                toPlot = Polygon.Polygon(c.contour(i)) & self.all
                if bool(toPlot):
                    for j in range(len(toPlot)):
                        #BoundPolyPoints = asarray(PolyUtils.pointList(toPlot.contour(j)))
                        BoundPolyPoints = asarray(PolyUtils.pointList(Polygon.Polygon(toPlot.contour(j))))
                        if self.operate_system == 2:
                            self.ax.plot(BoundPolyPoints[:,0],BoundPolyPoints[:,1],string,linewidth=w)
                            self.ax.plot([BoundPolyPoints[-1,0],BoundPolyPoints[0,0]],[BoundPolyPoints[-1,1],BoundPolyPoints[0,1]],string,linewidth=w)
                        else:
                            plt.plot(BoundPolyPoints[:,0],BoundPolyPoints[:,1],string,linewidth=w)
                            plt.plot([BoundPolyPoints[-1,0],BoundPolyPoints[0,0]],[BoundPolyPoints[-1,1],BoundPolyPoints[0,1]],string,linewidth=w)
                            plt.figure(1).canvas.draw()

    def data_gen(self):
        #self.ax.cla()
        for regionName,regionPoly in self.map.iteritems():
            self.plotPoly(regionPoly,'k')
        """
        #for i in range(len(self.V)):
        if shape(V)[1] <= 2:
            plt.plot(( V[1,shape(V)[1]-1],q_g[0,0]),( V[2,shape(V)[1]-1],q_g[1,0]),'b')
        else:
            plt.plot(( V[1,E[0,shape(E)[1]-1]], V[1,shape(V)[1]-1],q_g[0,0]),( V[2,E[0,shape(E)[1]-1]], V[2,shape(V)[1]-1],q_g[1,0]),'b')

        self.plotPoly(self.realRobot, 'r')
        self.plotPoly(self.robot, 'b')
        """
        pose = self.pose_handler.getPose()
        self.ax.plot(pose[0],pose[1],'bo')
        """
        self.ax.plot(self.q_g[0],self.q_g[1],'ro')
        self.plotPoly(self.overlap,'g')
        self.plotPoly(self.m_line,'b')
        """
        yield(pose[0],pose[1])
        """
        self.ax.plot(self.prev_follow[0],self.prev_follow[1],'ko')
        """

    def jplot(self):
        ani = animation.FuncAnimation(self.fig, self.scope.update, self.data_gen)
        plt.show()

class _Scope:
    def __init__(self, ax, motion, maxt=2, dt=0.02):
        self.i = 0
        self.ax = ax
        self.line, = self.ax.plot(1)
        self.ax.set_ylim(0, 1)
        self.motion = motion

    def update(self,data):
        (data1) = self.motion.data_gen()
        a = data1.next()
        self.line.set_data(a)
        self.ax.relim()
        self.ax.autoscale()
        return self.line,


"""
=======================
MNIST dataset benchmark
=======================

Benchmark on the MNIST dataset.  The dataset comprises 70,000 samples
and 784 features. Here, we consider the task of predicting
10 classes -  digits from 0 to 9 from their raw images. By contrast to the
covertype dataset, the feature space is homogenous.

Example of output :

    [..]
    Classification performance:
    ===========================
    Classifier               train-time   test-time   error-rat
    ------------------------------------------------------------
    Nystroem-SVM                105.07s       0.91s       0.0227
    ExtraTrees                   48.20s       1.22s       0.0288
    RandomForest                 47.17s       1.21s       0.0304
    SampledRBF-SVM              140.45s       0.84s       0.0486
    CART                         22.84s       0.16s       0.1214
    dummy                         0.01s       0.02s       0.8973

"""
from __future__ import division, print_function

# Author: Issam H. Laradji
#         Arnaud Joly <arnaud.v.joly@gmail.com>
# License: BSD 3 clause

import os
from time import time
import argparse
import numpy as np

from sklearn.datasets import fetch_mldata
from sklearn.datasets import get_data_home
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.dummy import DummyClassifier
from sklearn.externals.joblib import Memory
from sklearn.kernel_approximation import Nystroem
from sklearn.kernel_approximation import RBFSampler
from sklearn.metrics import zero_one_loss
from sklearn.pipeline import make_pipeline
from sklearn.svm import LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.utils import check_array

# Memoize the data extraction and memory map the resulting
# train / test splits in readonly mode
memory = Memory(os.path.join(get_data_home(), 'mnist_benchmark_data'),
                mmap_mode='r')

@memory.cache
def load_data(dtype=np.float32, order='F'):
    """Load the data, then cache and memmap the train/test split"""
    ######################################################################
    ## Load dataset
    print("Loading dataset...")
    data = fetch_mldata('MNIST original')
    X = check_array(data['data'], dtype=dtype, order=order)
    y = data["target"]

    # Normalize features
    X = X / 255

    ## Create train-test split (as [Joachims, 2006])
    print("Creating train-test split...")
    n_train = 60000
    X_train = X[:n_train]
    y_train = y[:n_train]
    X_test = X[n_train:]
    y_test = y[n_train:]

    return X_train, X_test, y_train, y_test

ESTIMATORS = {
    "dummy": DummyClassifier(),
    'CART': DecisionTreeClassifier(),
    'ExtraTrees': ExtraTreesClassifier(n_estimators=100),
    'RandomForest': RandomForestClassifier(n_estimators=100),
    'Nystroem-SVM':
    make_pipeline(Nystroem(gamma=0.015, n_components=1000), LinearSVC(C=100)),
    'SampledRBF-SVM':
    make_pipeline(RBFSampler(gamma=0.015, n_components=1000), LinearSVC(C=100))
}

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--classifiers', nargs="+",
                        choices=ESTIMATORS, type=str,
                        default=['ExtraTrees', 'Nystroem-SVM'],
                        help="list of classifiers to benchmark.")
    parser.add_argument('--n-jobs', nargs="?", default=1, type=int,
                        help="Number of concurrently running workers for "
                             "models that support parallelism.")
    parser.add_argument('--order', nargs="?", default="C", type=str,
                        choices=["F", "C"],
                        help="Allow to choose between fortran and C ordered "
                             "data")
    parser.add_argument('--random-seed', nargs="?", default=0, type=int,
                        help="Common seed used by random number generator.")
    args = vars(parser.parse_args())

    print(__doc__)

    X_train, X_test, y_train, y_test = load_data(order=args["order"])

    print("")
    print("Dataset statistics:")
    print("===================")
    print("%s %d" % ("number of features:".ljust(25), X_train.shape[1]))
    print("%s %d" % ("number of classes:".ljust(25), np.unique(y_train).size))
    print("%s %s" % ("data type:".ljust(25), X_train.dtype))
    print("%s %d (size=%dMB)" % ("number of train samples:".ljust(25),
                                 X_train.shape[0], int(X_train.nbytes / 1e6)))
    print("%s %d (size=%dMB)" % ("number of test samples:".ljust(25),
                                  X_test.shape[0], int(X_test.nbytes / 1e6)))

    print()
    print("Training Classifiers")
    print("====================")
    error, train_time, test_time = {}, {}, {}
    for name in sorted(args["classifiers"]):
        print("Training %s ... " % name, end="")
        estimator = ESTIMATORS[name]
        estimator_params = estimator.get_params()

        estimator.set_params(**{p: args["random_seed"]
                                for p in estimator_params
                                if p.endswith("random_state")})

        if "n_jobs" in estimator_params:
            estimator.set_params(n_jobs=args["n_jobs"])

        time_start = time()
        estimator.fit(X_train, y_train)
        train_time[name] = time() - time_start

        time_start = time()
        y_pred = estimator.predict(X_test)
        test_time[name] = time() - time_start

        error[name] = zero_one_loss(y_test, y_pred)

        print("done")

    print()
    print("Classification performance:")
    print("===========================")
    print("{0: <24} {1: >10} {2: >11} {3: >12}"
          "".format("Classifier  ", "train-time", "test-time", "error-rate"))
    print("-" * 60)
    for name in sorted(args["classifiers"], key=error.get):

        print("{0: <23} {1: >10.2f}s {2: >10.2f}s {3: >12.4f}"
              "".format(name, train_time[name], test_time[name], error[name]))

    print()


import sys
import parsetags
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn import svm
from sklearn.decomposition import PCA as PCA
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import nltk
from nltk.stem import LancasterStemmer
import re

def get_top_tags(segment_tags, n):
	tag_freqs = {}
	for tag_list in segment_tags.values():
		for tag in tag_list:
			if tag not in tag_freqs:
				tag_freqs[tag] = 0
			tag_freqs[tag] += 1
	return ['NULL'] + sorted(tag_freqs.keys(), key=lambda x: tag_freqs[x])[-n:]

def get_common_words(n=100):
	try:
		file_content = open(sys.argv[3]).read()
		common_words = nltk.word_tokenize(file_content)
	except IndexError:
		return None
	return set(common_words[:n])

def get_named_entities():
	try:
		file_content = open(sys.argv[2]).read()
		named_entities = nltk.word_tokenize(file_content)
	except IndexError:
		return None
	return set(named_entities)

def filter_segments(segment_tags, ntags):
	filtered_segtags = {}
	for segment in segment_tags:
		# np.random.shuffle(segment_tags[segment])
		for tag in segment_tags[segment]:
			if tag not in ntags: continue
			filtered_segtags[segment] = ntags.index(tag)
		if segment not in filtered_segtags:
			filtered_segtags[segment] = 0
	return filtered_segtags

def increase_num_segments(segment_tags, n, length=1000):
	new_segment_tags = {}
	segments = sorted(segment_tags.keys(), key=len)
	lengths = np.array([len(seg) for seg in segments])
	dist = lengths/np.sum(lengths)
	random_segments = np.random.choice(segments, size=n, p=dist)
	for segment in random_segments:
		new_segment = segment
		if len(new_segment) > length:
			index = np.random.randint(0, len(new_segment)-length)
			new_segment = new_segment[index:index+length]
		new_segment_tags[new_segment] = segment_tags[segment]
	return new_segment_tags

def named_entity_reduction(segment_tags, named_entities, common_words):
	punctuation = [',', '.', "'", '?', ';', ':', '!', '(', ')', '`', '--'
			   '\xe2', '\x80', '\x94', '\x99']
	new_segments = []
	segments = list(segment_tags.keys())
	for segment in segments:
		new_segment = ''
		tokens = nltk.word_tokenize(segment)
		for token in tokens:
			if token in punctuation: continue
			if token.lower() in common_words: continue
			if token not in named_entities: continue
			new_segment += token + ' '
		new_segments.append(new_segment)
	new_segment_tags = {}
	for i in range(len(segments)):
		new_segment_tags[new_segments[i]] = segment_tags[segments[i]]
	return new_segment_tags

def stemming_reduction(segment_tags):
	punctuation = [',', '.', "'", '?', ';', ':', '!', '(', ')', '`', '--'
					'\xe2', '\x80', '\x94', '\x99']
	new_segments = []
	stemmer = LancasterStemmer()
	segments = list(segment_tags.keys())
	for segment in segments:
		new_segment = ''
		segment = re.sub(r'[^\x00-\x7f]',r'', segment)
		tokens = nltk.word_tokenize(segment)
		for token in tokens:
			if token in punctuation: continue
			try:
				new_segment += stemmer.stem(token)+' '
			except UnicodeDecodeError:
				new_segment += ''
		new_segments.append(new_segment)
	stemmed_segment_tags = {}
	for i in range(len(segments)):
		stemmed_segment_tags[new_segments[i]] = segment_tags[segments[i]]
	return stemmed_segment_tags

def separate_segments(segment_tags, k):
	train = {}
	for segment in segment_tags.keys():
		if np.random.random() < k:
			train[segment] = segment_tags.pop(segment)
	return train, segment_tags

def bag_of_words(segment_tags, tfidf=False):
	#create matrix of word frequencies
	segments = list(segment_tags.keys())
	vec = CountVectorizer()
	word_freqs = vec.fit_transform(segments).toarray()
	if tfidf:
		tfidf_transformer = TfidfTransformer()
		word_freqs = tfidf_transformer.fit_transform(word_freqs)
	labels = np.empty(shape=len(segments))
	for i in range(len(segments)):
		labels[i] = segment_tags[segments[i]]
	return word_freqs, labels, segments

def entity_bow(segment_tags, named_entities, common_words):
	punctuation = [',', '.', "'", '?', ';', ':', '!', '(', ')', '`', '--'
				   '\xe2', '\x80', '\x94', '\x99']
	new_segments = []
	segments = list(segment_tags.keys())
	for segment in segments:
		new_segment = ''
		tokens = nltk.word_tokenize(segment)
		for token in tokens:
			if token in punctuation: continue
			if token.lower() in common_words: continue
			if token not in named_entities: continue
			new_segment += token + ' '
		new_segments.append(new_segment)

	vec = CountVectorizer()
	word_freqs = vec.fit_transform(new_segments).toarray()
	tfidf_transformer = TfidfTransformer()
	X_train_tfidf = tfidf_transformer.fit_transform(word_freqs)
	print(word_freqs.shape, X_train_tfidf.shape)
	labels = np.empty(shape=len(segments))
	for i in range(len(segments)):
		labels[i] = segment_tags[segments[i]]
	return X_train_tfidf, labels, segments

def pca_plot(Xtrain, ytrain):
	#binary classification case
	X_reduced = Xtrain
	pca = PCA(3)
	X_pca = pca.fit_transform(X_reduced)
	ax = plt.axes(projection='3d')
	for i in range(X_pca.shape[0]):
		if ytrain[i] == 1:
			ax.scatter(X_pca[i, 0], X_pca[i, 2], X_pca[i, 1], 'o', color='blue')
		else:
			ax.scatter(X_pca[i, 0], X_pca[i, 2], X_pca[i,1], 'x', color='red')
	plt.show()

def randomize_order(X, y, segments):
	shuffled_segments = []
	indices = np.arange(len(segments))
	np.random.shuffle(indices)
	X, y = X[indices], y[indices]
	for i in indices:
		shuffled_segments.append(segments[i])
	return X, y, segments

def naive_bayes(segment_tags, k=0.5, normalize=False):
	X, y, segments = randomize_order(*bag_of_words(segment_tags, tfidf=normalize))
	num_examples = len(segments)
	Xtest, ytest = X[int(k*num_examples):, :], y[int(k*num_examples):]
	Xtrain, ytrain = X[:int(k*num_examples), :], y[:int(k*num_examples)]
	nb_classifier = MultinomialNB().fit(Xtrain, ytrain)
	nb_predicted_tags = nb_classifier.predict(Xtest)
	nb_success_rate = np.mean(nb_predicted_tags == ytest)
	return 1-nb_success_rate

def support_vector(segment_tags, k=0.5, normalize=False):
	X, y, segments = randomize_order(*bag_of_words(segment_tags, tfidf=normalize))
	num_examples = len(segments)
	Xtest, ytest = X[int(k*num_examples):, :], y[int(k*num_examples):]
	Xtrain, ytrain = X[:int(k*num_examples), :], y[:int(k*num_examples)]
	svm_classifier = svm.SVC()
	svm_classifier.fit(Xtrain, ytrain)
	svm_predicted_tags = svm_classifier.predict(Xtest)
	svm_success_rate = np.mean(svm_predicted_tags == ytest)
	return 1-svm_success_rate

if __name__ == "__main__":
	orig_segment_tags = parsetags.parse_tags(sys.argv[1])
	common_words = get_common_words()
	named_entities = get_named_entities()
	for sample_size in ['BIG']:
		if sample_size == 'BIG':
			segment_tags = increase_num_segments(orig_segment_tags, 3000, length=1000)
		orig_segment_tags = segment_tags
		for text_features in ['REGULAR', 'STEMMED', 'NAMED']:
			if text_features == 'STEMMED':
				segment_tags = stemming_reduction(orig_segment_tags)
			if text_features == 'NAMED':
				segment_tags = named_entity_reduction(orig_segment_tags, named_entities, common_words)
			for freq_feature in ['COUNT', 'TFIDF']:
				# ntags = get_top_tags(segment_tags, 7)
				print(sample_size, text_features, freq_feature)
				ntags = ['ETA', 'EHDRH', 'AFR']
				filtered_segtags = filter_segments(segment_tags, ntags)
				with open('Results/' + sample_size + '_' + text_features + '_' + freq_feature + '.txt', 'w') as f:
					for i in range(100):
						f.write(str(naive_bayes(filtered_segtags, normalize=(freq_feature is 'TFIDF'))) + '\n')

	# segment_tags = parsetags.parse_tags(sys.argv[1])
	# big_segment_tags = increase_num_segments(segment_tags, 3000, length=1000)
	# ntags = get_top_tags(segment_tags, 7)
	# for
	# # ntags = ['NULL', 'ETA', 'EHDRH', 'AFR']

	# common_words = get_common_words()
	# named_entities = get_named_entities()
	# filtered_segtags = filter_segments(segment_tags, ntags)
	# #entity_bow(filtered_segtags, named_entities, common_words)
	# naive_bayes(filtered_segtags, named_entities, common_words, features=entity_bow)
	# naive_bayes(filtered_segtags)
	# support_vector(filtered_segtags)

	# predicted_tags = [ntags[int(np.round(nb_predicted_tags[i]))] for i in range(len(svm_predicted_tags))]
	# count = 0
	# print(ntags)
	# for i in range(len(predicted_tags)):
	# 	if predicted_tags[i] == 'NULL':
	# 		if all(tag not in segment_tags[shuffled_segments[i]] for tag in ntags):
	# 			count += 1
	# 	else:
	# 		if predicted_tags[i] in segment_tags[shuffled_segments[i]]:
	# 			count += 1
	# print(count/len(predicted_tags))