"""
Testing for Theil-Sen module (sklearn.linear_model.theil_sen)
"""

# Author: Florian Wilhelm <florian.wilhelm@gmail.com>
# License: BSD 3 clause

from __future__ import division, print_function, absolute_import

import os
import sys
from contextlib import contextmanager
import numpy as np
from numpy.testing import assert_array_equal, assert_array_less
from numpy.testing import assert_array_almost_equal, assert_warns
from scipy.linalg import norm
from scipy.optimize import fmin_bfgs
from nose.tools import raises, assert_almost_equal
from sklearn.utils import ConvergenceWarning
from sklearn.linear_model import LinearRegression, TheilSenRegressor
from sklearn.linear_model.theil_sen import _spatial_median, _breakdown_point
from sklearn.linear_model.theil_sen import _modified_weiszfeld_step
from sklearn.utils.testing import assert_greater, assert_less

@contextmanager
def no_stdout_stderr():
    old_stdout = sys.stdout
    old_stderr = sys.stderr
    sys.stdout = open(os.devnull, 'w')
    sys.stderr = open(os.devnull, 'w')
    yield
    sys.stdout.flush()
    sys.stderr.flush()
    sys.stdout = old_stdout
    sys.stderr = old_stderr

def gen_toy_problem_1d(intercept=True):
    random_state = np.random.RandomState(0)
    # Linear model y = 3*x + N(2, 0.1**2)
    w = 3.
    if intercept:
        c = 2.
        n_samples = 50
    else:
        c = 0.1
        n_samples = 100
    x = random_state.normal(size=n_samples)
    noise = 0.1 * random_state.normal(size=n_samples)
    y = w * x + c + noise
    # Add some outliers
    if intercept:
        x[42], y[42] = (-2, 4)
        x[43], y[43] = (-2.5, 8)
        x[33], y[33] = (2.5, 1)
        x[49], y[49] = (2.1, 2)
    else:
        x[42], y[42] = (-2, 4)
        x[43], y[43] = (-2.5, 8)
        x[53], y[53] = (2.5, 1)
        x[60], y[60] = (2.1, 2)
        x[72], y[72] = (1.8, -7)
    return x[:, np.newaxis], y, w, c

def gen_toy_problem_2d():
    random_state = np.random.RandomState(0)
    n_samples = 100
    # Linear model y = 5*x_1 + 10*x_2 + N(1, 0.1**2)
    X = random_state.normal(size=(n_samples, 2))
    w = np.array([5., 10.])
    c = 1.
    noise = 0.1 * random_state.normal(size=n_samples)
    y = np.dot(X, w) + c + noise
    # Add some outliers
    n_outliers = n_samples // 10
    ix = random_state.randint(0, n_samples, size=n_outliers)
    y[ix] = 50 * random_state.normal(size=n_outliers)
    return X, y, w, c

def gen_toy_problem_4d():
    random_state = np.random.RandomState(0)
    n_samples = 10000
    # Linear model y = 5*x_1 + 10*x_2  + 42*x_3 + 7*x_4 + N(1, 0.1**2)
    X = random_state.normal(size=(n_samples, 4))
    w = np.array([5., 10., 42., 7.])
    c = 1.
    noise = 0.1 * random_state.normal(size=n_samples)
    y = np.dot(X, w) + c + noise
    # Add some outliers
    n_outliers = n_samples // 10
    ix = random_state.randint(0, n_samples, size=n_outliers)
    y[ix] = 50 * random_state.normal(size=n_outliers)
    return X, y, w, c

def test_modweiszfeld_step_1d():
    X = np.array([1., 2., 3.]).reshape(3, 1)
    # Check startvalue is element of X and solution
    median = 2.
    new_y = _modified_weiszfeld_step(X, median)
    assert_array_almost_equal(new_y, median)
    # Check startvalue is not the solution
    y = 2.5
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_less(median, new_y)
    assert_array_less(new_y, y)
    # Check startvalue is not the solution but element of X
    y = 3.
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_less(median, new_y)
    assert_array_less(new_y, y)
    # Check that a single vector is identity
    X = np.array([1., 2., 3.]).reshape(1, 3)
    y = X[0, ]
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_equal(y, new_y)

def test_modweiszfeld_step_2d():
    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
    y = np.array([0.5, 0.5])
    # Check first two iterations
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_almost_equal(new_y, np.array([1 / 3, 2 / 3]))
    new_y = _modified_weiszfeld_step(X, new_y)
    assert_array_almost_equal(new_y, np.array([0.2792408, 0.7207592]))
    # Check fix point
    y = np.array([0.21132505, 0.78867497])
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_almost_equal(new_y, y)

def test_spatial_median_1d():
    X = np.array([1., 2., 3.]).reshape(3, 1)
    true_median = 2.
    _, median = _spatial_median(X)
    assert_array_almost_equal(median, true_median)
    # Test larger problem and for exact solution in 1d case
    random_state = np.random.RandomState(0)
    X = random_state.randint(100, size=(1000, 1))
    true_median = np.median(X.ravel())
    _, median = _spatial_median(X)
    assert_array_equal(median, true_median)

def test_spatial_median_2d():
    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
    _, median = _spatial_median(X, max_iter=100, tol=1.e-6)

    def cost_func(y):
        dists = np.array([norm(x - y) for x in X])
        return np.sum(dists)

    # Check if median is solution of the Fermat-Weber location problem
    fermat_weber = fmin_bfgs(cost_func, median, disp=False)
    assert_array_almost_equal(median, fermat_weber)
    # Check when maximum iteration is exceeded a warning is emitted
    assert_warns(ConvergenceWarning, _spatial_median, X, max_iter=30, tol=0.)

def test_theil_sen_1d():
    X, y, w, c = gen_toy_problem_1d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(np.abs(lstq.coef_ - w), 0.9)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_theil_sen_1d_no_intercept():
    X, y, w, c = gen_toy_problem_1d(intercept=False)
    # Check that Least Squares fails
    lstq = LinearRegression(fit_intercept=False).fit(X, y)
    assert_greater(np.abs(lstq.coef_ - w - c), 0.5)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(fit_intercept=False,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w + c, 1)
    assert_almost_equal(theil_sen.intercept_, 0.)

def test_theil_sen_2d():
    X, y, w, c = gen_toy_problem_2d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(norm(lstq.coef_ - w), 1.0)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(max_subpopulation=1e3,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_calc_breakdown_point():
    bp = _breakdown_point(1e10, 2)
    assert_less(np.abs(bp - 1 + 1/(np.sqrt(2))), 1.e-6)

@raises(ValueError)
def test_checksubparams_negative_subpopulation():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(max_subpopulation=-1, random_state=0).fit(X, y)

@raises(ValueError)
def test_checksubparams_too_few_subsamples():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(n_subsamples=1, random_state=0).fit(X, y)

@raises(ValueError)
def test_checksubparams_too_many_subsamples():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(n_subsamples=101, random_state=0).fit(X, y)

@raises(ValueError)
def test_checksubparams_n_subsamples_if_less_samples_than_features():
    random_state = np.random.RandomState(0)
    n_samples, n_features = 10, 20
    X = random_state.normal(size=(n_samples, n_features))
    y = random_state.normal(size=n_samples)
    TheilSenRegressor(n_subsamples=9, random_state=0).fit(X, y)

def test_subpopulation():
    X, y, w, c = gen_toy_problem_4d()
    theil_sen = TheilSenRegressor(max_subpopulation=250,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_subsamples():
    X, y, w, c = gen_toy_problem_4d()
    theil_sen = TheilSenRegressor(n_subsamples=X.shape[0],
                                  random_state=0).fit(X, y)
    lstq = LinearRegression().fit(X, y)
    # Check for exact the same results as Least Squares
    assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 9)

def test_verbosity():
    X, y, w, c = gen_toy_problem_1d()
    # Check that Theil-Sen can be verbose
    with no_stdout_stderr():
        TheilSenRegressor(verbose=True, random_state=0).fit(X, y)
        TheilSenRegressor(verbose=True,
                          max_subpopulation=10,
                          random_state=0).fit(X, y)

def test_theil_sen_parallel():
    X, y, w, c = gen_toy_problem_2d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(norm(lstq.coef_ - w), 1.0)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(n_jobs=-1,
                                  random_state=0,
                                  max_subpopulation=2e3).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_less_samples_than_features():
    random_state = np.random.RandomState(0)
    n_samples, n_features = 10, 20
    X = random_state.normal(size=(n_samples, n_features))
    y = random_state.normal(size=n_samples)
    # Check that Theil-Sen falls back to Least Squares if fit_intercept=False
    theil_sen = TheilSenRegressor(fit_intercept=False,
                                  random_state=0).fit(X, y)
    lstq = LinearRegression(fit_intercept=False).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 12)
    # Check fit_intercept=True case. This will not be equal to the Least
    # Squares solution since the intercept is calculated differently.
    theil_sen = TheilSenRegressor(fit_intercept=True, random_state=0).fit(X, y)
    y_pred = theil_sen.predict(X)
    assert_array_almost_equal(y_pred, y, 12)


"""
88b           d88  88               88
888b         d888  ""               88
88`8b       d8'88                   88
88 `8b     d8' 88  88  8b,dPPYba,   88   ,d8   ,adPPYba,
88  `8b   d8'  88  88  88P'   `"8a  88 ,a8"   a8P_____88
88   `8b d8'   88  88  88       88  8888[     8PP"""""""
88    `888'    88  88  88       88  88`"Yba,  "8b,   ,aa
88     `8'     88  88  88       88  88   `Y8a  `"Ybbd8"'

--------------------------------------------------------

This file is a part of Minke, a tool for generating simulated
gravitational wave signals, used for characterising and training
search algorithms.

Minke was created by Daniel Williams, based on work started by Chris
Pankow and others, and is built around the LALSimulation library.

"""
from glue.ligolw import ligolw, utils, lsctables
lsctables.use_in(ligolw.LIGOLWContentHandler);
import numpy
import lalburst, lalsimulation, lalmetaio
from minke.antenna import response

from lal import TimeDelayFromEarthCenter as XLALTimeDelayFromEarthCenter
#from pylal.xlal.datatypes.ligotimegps import LIGOTimeGPS
from lal import LIGOTimeGPS
from glue.ligolw.utils import process
import glue

import glue.ligolw
import gzip

import lal, lalframe

import numpy as np
import pandas as pd
import os
import os.path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import re
import random
import minke
from minke import sources
sourcemap = {}
for classin in dir(sources):
    classin = sources.__dict__[classin]
    if hasattr(classin, "waveform"):
        sourcemap[classin.waveform] = classin

def source_from_row(row):
    waveform = row.waveform
    sourceobj = sourcemap[row.waveform].__new__(sourcemap[row.waveform])
    sourceobj.numrel_data = str("")
    params = {}
    for attr in dir(row):
        if not attr[0] == "_" and not attr[:3] =="get":
            #print attr
            try:
                params[attr] = getattr(row, attr)
                setattr(sourceobj, attr, getattr(row, attr))
            except AttributeError:
                print("Error processing the {} column".format(attr))
    sourceobj.params = params
    try:
        sourceobj.time = row.time_geocent_gps
    except:
        sourceobj.time = row.geocent_start_time
        pass
    return sourceobj

def source_from_dict(params):
    sourceobj = sourcemap[params['morphology']].__new__(sourcemap[params['morphology']])
    sourceobj.numrel_data = str("")
    params = {}
    for attr in dir(row):
        if not attr[0] == "_" and not attr[:3] =="get":
            #print attr
            params[attr] = getattr(row, attr)
            setattr(sourceobj, attr, getattr(row, attr))
    sourceobj.params = params
    try:
        sourceobj.time = row.time_geocent_gps
    except:
        sourceobj.time = row.geocent_start_time
        pass
    return sourceobj

table_types = {
    # Ad-Hoc
    "ga" : lsctables.SimBurstTable,
    "sg" : lsctables.SimBurstTable,
    "wnb" : lsctables.SimBurstTable,
    "sc" : lsctables.SimBurstTable,
    # Supernova Families
    "d08" : lsctables.SimBurstTable,
    "s10" : lsctables.SimBurstTable,
    "m12" : lsctables.SimBurstTable,
    "o13" : lsctables.SimBurstTable,
    "y10" : lsctables.SimBurstTable,
    # Long Duration
    "adi" : lsctables.SimBurstTable,
    # Ringdown
    "rng" : lsctables.SimRingdownTable,
    "gng" : lsctables.SimRingdownTable,
    }

tables = {
    "burst" : lsctables.SimBurstTable,
    "ringdown" : lsctables.SimRingdownTable
    }

def mkdir(path):
    """
    Make all of the tree of directories in a given path if they don't
    already exist.

    Parameters
    ----------
    path : str
       The path to the desired directory.

    """
    sub_path = os.path.dirname(path)
    if not os.path.exists(sub_path):
        mkdir(sub_path)
    if not os.path.exists(path):
        os.mkdir(path)

class TableTypeError(Exception):
    pass

class MDCSet():

    inj_families_names = {'ga' : 'Gaussian',
                          'sg' : 'SineGaussian',
                          'wnb': 'BTLWNB',
                          "sc" : "StringCusp",
                          # Supernova families
                          'd08' : 'Dimmelmeier+08',
                          's10' : 'Scheidegger+10',
                          'm12' : 'Mueller+12',
                          'o13' : 'Ott+13',
                          'y10' : "Yakunin+10",
                          # Long-duration
                          'adi' : 'ADI',
                          # Ringdown
                          'rng' : "BBHRingdown",
                          'gng' : "GenericRingdown",
                          }

    inj_families_abb = dict((v,k) for k,v in list(inj_families_names.items()))

    hist_parameters = {
        "StringCusp": ["amplitude", "ra", "dec"],
        "SineGaussian": ["hrss", "psi", "ra", "dec"],
        "Gaussian": ["hrss", "psi", "ra", "dec"],
        "BTLWNB": ["hrss", "ra", "dec"],
        "Dimmelmeier+08": ['hrss', 'ra', 'dec']
    }

    waveforms = []

    def __init__(self, detectors, name='MDC Set', table_type = "burst"):
        """
        Represents an MDC set, stored in an XML SimBurstTable file.

        Parameters
        ----------
        detectors : list
            A list of detector names where the injections should be made.

        name : str
            A name for the MDC Set. Defaults to 'MDC Set'.

        table_type : str
            The type of table which should be generated. Default is `burst`,
            which generates a SimBurstTable.
        """
        self.detectors = detectors

        self.waveforms = []
        self.strains = []
        self.egw = []
        self.times = []
        self.name = name

        self.times = np.array(self.times)

        self.table_type = tables[table_type]

    def __add__(self, waveform):
        """
        Handle a waveform being added to the MDC set.

        Parameters
        ----------
        waveform : Waveform object
           The waveform which should be added to the MDC set.

        """

        # Check that this type of waveform can go into this type of
        # XML file.
        if not table_types[self.inj_families_abb[waveform.waveform]] == self.table_type:
            raise TableTypeError()

        self.waveforms.append(waveform)
        self.times = np.append(self.times, waveform.time)

    def save_xml(self, filename):
        """
        Save the MDC set as an XML SimBurstTable.

        Parameters
        ----------
        filename : str
           The location to save the xml file. The output is gzipped, so ending it with
           a ".gz" would stick with convention.
        """
        xmldoc = ligolw.Document()
        lw = xmldoc.appendChild(ligolw.LIGO_LW())
        sim = lsctables.New(self.table_type)
        lw.appendChild(sim)
        # This needs to be given the proper metadata once the package has the maturity to
        # write something sensible.
        for waveform in self.waveforms:
            procrow = process.register_to_xmldoc(xmldoc, "minke_burst_mdc+{}".format(minke.__version__), {}) # waveform.params)
            try:
                waveform_row = waveform._row(sim)
                waveform_row.process_id = procrow.process_id
            except:
                row = sim.RowType()
                for a in list(self.table_type.validcolumns.keys()):
                    if a in list(waveform.params.keys()):
                        setattr(row, a, waveform.params[a])
                    else:
                        if not hasattr(waveform, a):
                            setattr(row, a, 0)
                        else:
                            setattr(row, a, getattr(waveform, a))

                row.waveform = waveform.waveform
                if self.table_type == lsctables.SimBurstTable:
                    # Fill in the time
                    row.set_time_geocent(GPS(float(waveform.time)))
                    # Get the sky locations
                    row.ra, row.dec, row.psi = waveform.ra, waveform.dec, waveform.psi
                row.simulation_id = waveform.simulation_id
                row.waveform_number = random.randint(0,int(2**32)-1)
                ### !! This needs to be updated.
                row.process_id = "process:process_id:0" #procrow.process_id

                waveform_row = row

            sim.append(waveform_row)
            #del waveform_row
        # Write out the xml and gzip it.
        utils.write_filename(xmldoc, filename, gz=True)

    def load_xml(self, filename, full=True, start=None, stop=None):
        """Load the MDC Set from an XML file containing the SimBurstTable.

        Parameters
        ----------
        filename : str
           The filename of the XML file.

        full : bool
           If this is true (which is the default) then all of
           the calculated parameters are computed from the waveform
           definintion.

        start : float
           The time at which the xml read-in should
           start. The default is "None", in which case the xml file
           will be read-in from the start.

        end : float
           The last time to be read from the xml file. The default is None,
           which causes the xml to be read right-up to the last time in the
           file.

        To Do
        -----
        A the moment this loads the information in to the object, but it
        doesn't produce waveform objects for each of the injections in the
        file. This should be fixed so that the object works symmetrically.
        """
        i = 0
        #sim_burst_table = lalburst.SimBurstTableFromLIGOLw(filename, start, stop)

        xml = glue.ligolw.utils.load_filename(filename,
                                              contenthandler = glue.ligolw.ligolw.LIGOLWContentHandler,
                                              verbose = True)
        sim_burst_table = glue.ligolw.table.get_table(xml, self.table_type.tableName)

        for i,simrow in enumerate(sim_burst_table):
            # This is an ugly kludge to get around the poor choice of wavform name in the xmls, and
            if simrow.waveform[:3]=="s15":
                self.numrel_file = str(sim_burst_table.waveform)
                sim_burst_table.waveform = "Dimmelmeier+08"

            self.waveforms.append(source_from_row(simrow))

            if full:
                self._measure_hrss(i)
                self._measure_egw_rsq(i)

            if self.table_type == tables["burst"]:
                self.times = np.append(self.times, float(simrow.time_geocent_gps))

    def _generate_burst(self,row,rate=16384.0):
        """
        Generate the burst described in a given row, so that it can be
        measured.

        Parameters
        ----------
        row : SimBurst Row
            The row of the waveform to be measured
        rate : float
            The sampling rate of the signal, in Hz. Defaults to 16384.0Hz

        Returns
        -------
        hp :
            The strain in the + polarisation
        hx :
            The strain in the x polarisation
        hp0 :
            A copy of the strain in the + polarisation
        hx0 :
            A copy of the strain in the x polarisation
        """
        row = self.waveforms[row]
        hp, hx, hp0, hx0 = row._generate()
        return hp, hx, hp0, hx0

    def _getDetector(self, det):
        """
        A method to return a LALDetector object corresponding to a detector's
        X#-style name, e.g. 'H1' as the Hanford 4km detector.

        Parameters
        ----------
        det : str
            A string describing the detector in the format letter-number, e.g
            "H1" would be the Hanford 4km detector, "L1" would be the
            Livingston 4km, and so-forth.

        Returns
        -------
        detector : LALDetector
            The LAL object describing the detector
        """
        # get detector
        return lalsimulation.DetectorPrefixToLALDetector(det)
        #if det not in lal.cached_detector_by_prefix.keys():
        #      raise ValueError, "%s is not a cached detector.  "\
        #            "Cached detectors are: %s" % (det, inject.cached_detector.keys())
        #return lal.cached_detector_by_prefix[det]

    def _timeDelayFromGeocenter(self, detector, ra, dec, gpstime):
        """
        Calculate the time delay between the geocentre and a given detector
        for a signal from some sky location.

        Parameters
        ----------
        detector : str
            A string describing the detector, e.g. H1 is the Hanford 4km
            detector.
        ra : float
            The right-ascension of the observation in radians
        dec : float
            The declination of the obser
        """
        if isinstance(detector, str): detector = self._getDetector(detector)
        gpstime = LIGOTimeGPS(float(gpstime))
        return XLALTimeDelayFromEarthCenter(detector.location, ra, dec, gpstime)

    def directory_path(self):
        """
        Generate the directory where the frames from this MDC should be stored,
        so, e.g. Gaussians 0d100 would go in "ga/ga0d100/"

        Returns
        -------
        str
           the folder structure
        """
        name = self._simID(0)
        abb = self.inj_families_abb[self.waveforms[0].waveform].lower()
        return "{}/{}".format(abb, name)

    def _simID(self, row):
        """
        Generate a name for an injection set in the format expected by cWB

        Parameters
        ----------
        row : SimBurst
            The simburst table row describing the injection

        Returns
        -------
        str
           The name of the injection in the cWB format
        """
        row = self.waveforms[row]

        name = ''
        numberspart = ''
        if row.waveform in ("Dimmelmeier+08", "Scheidegger+10", "Mueller+12", "Ott+13", "Yakunin+10"):
            #print row
            numberspart = os.path.basename(row.params['numrel_data']).split('.')[0]

        if row.waveform == "Gaussian":
            numberspart = "{:.3f}".format(row.duration * 1e3)
        elif row.waveform == "SineGaussian":
            if row.pol_ellipse_e==1.0:
                pol="linear"
            elif row.pol_ellipse_e==0.0:
                pol="circular"
            elif 0.0<row.pol_ellipse_e<1.0:
                pol = "elliptical"
            else:
                pol = "inclined"
            numberspart = "f{:.0f}_q{:.0f}_{}".format(row.frequency, row.q, pol)
        elif row.waveform == "BTLWNB":
            numberspart = "{}b{}tau{}".format(row.frequency, row.bandwidth, row.duration)
        name += '{}_{}'.format(self.inj_families_abb[row.waveform].lower(), numberspart).replace('.','d')

        return name

    def _measure_hrss(self, row, rate=16384.0):
        """
        Measure the various components of hrss (h+^2, hx^2, hphx) for a given
        input row. This is accomplished by generating the burst and calling
        the SWIG wrapped  XLALMeasureHrss in lalsimulation.

        Parameters
        ----------
        row : int
            The row number of the waveforms to be measured
        rate : float
            The sampling rate of the signal, in Hz. Defaults to 16384.0Hz

        Returns
        -------
        hrss : float
            The measured hrss of the waveform amplitude: sqrt(|Hp|^2 + |Hx|^2)
        hphp : float
            The hrss of the + polarisation only.
        hxhx : float
            The hrss of the x polarisation only.
        hphx : float
            The hrss of |HpHx|
        """
        row = self.waveforms[row]
        hp, hx, hp0, hx0 = row._generate() #self._generate_burst(row)# self.hp, self.hx, self.hp0, self.hx0

        hp0.data.data *= 0
        hx0.data.data *= 0

        # H+ hrss only
        hphp = lalsimulation.MeasureHrss(hp, hx0)**2
        # Hx hrss only
        hxhx = lalsimulation.MeasureHrss(hp0, hx)**2
        # sqrt(|Hp|^2 + |Hx|^2)
        hrss = lalsimulation.MeasureHrss(hp, hx)

        hp.data.data = numpy.abs(hx.data.data) + numpy.abs(hp.data.data)
        # |H+Hx|
        hphx = (lalsimulation.MeasureHrss(hp, hx0)**2 - hrss**2)/2
        #print hrss
        self.strains.append([hrss, hphp, hxhx, hphx])

    def _measure_egw_rsq(self, row,  rate=16384.0):
        """
        Measure the energy emitted in gravitational waves divided
        by the distance squared in M_solar / pc^2. This is accomplished
        by generating the burst and calling the SWIG wrapped
        XLALMeasureHrss in lalsimulation.

        Parameters
        ----------
        row : int
            The row number of the waveforms to be measured
        rate : float
            The sampling rate of the signal, in Hz. Defaults to 16384.0Hz

        Returns
        -------
        egw : float
            The energy emitted in gravitational waves divided
            by the distance squared in M_solar / pc^2.
        """
        hp, hx, _, _ =  self._generate_burst(row)
        self.egw.append(lalsimulation.MeasureEoverRsquared(hp, hx))

    def _responses(self, row):
        """
        Calculate the antenna repsonses for each detector to the waveform.

        Parameters
        ----------
        row : int
            The row number of the waveforms to be measured

        Returns
        -------
        responses : list of lists
            A list containing the lists of antenna responses, with the first
            element of each list containing the detector acronym.
        """
        output = []
        row = self.waveforms[row]
        for detector in self.detectors:
            time = row.time_geocent_gps + self._timeDelayFromGeocenter(detector, row.ra, row.dec, row.time_geocent_gps)
            time = np.float64(time)
            rs = response(time, row.ra, row.dec, 0, row.psi, 'radians', detector)
            output.append([detector, time, rs[0], rs[1]]   )
        return output

    def plot_skymap(self):
        """
        Plot a skymap of the injections distribution in RA and DEC on a Hammer projection.

        Returns
        -------
        matplotlib figure
        """
        fig = plt.figure()
        # Load the ra and dec numbers out of the waveforms
        dec = [getattr(s, 'dec') for s in self.waveforms]
        ra = [getattr(s, 'ra') for s in self.waveforms]

        # Make the plot on a hammer projection
        plt.subplot(111, projection='hammer')
        H, x, y = np.histogram2d(ra, dec, [50, 25], range=[[0, 2*np.pi], [-np.pi/2, np.pi/2]])
        dist = plt.pcolormesh(x-np.pi,y, H.T, cmap="viridis")
        plt.title("Sky distribution")
        plt.colorbar(dist, orientation='horizontal')
        return fig

    def plot_hist(self, parameter):
        """
        Plot a histogram of a waveform parameter.

        Parameters
        ----------
        parameter : str
           The name of the simburst table parameter which is desired for the plot.

        Returns
        -------
        matplotlib figure
        """
        fig = plt.figure()
        prms = [getattr(s, parameter) for s in self.waveforms]
        ax2 = plt.subplot(111)
        ax2.set_title("{} distribution".format(parameter))
        ax2.set_xlabel(parameter)
        ax2.hist(prms, bins=100, log=True, histtype="stepfilled", alpha=0.6);
        return fig

    def gravEn_row(self, row, frame):
        """
        Produces a gravEn-style log row for a row of the simBurstTable.

        Parameters
        ----------
        row : int
            The row number of the waveforms to be measured

        Returns
        -------
        str
            A string in the gravEn format which describes the injection.
        """
        strains = self.strains[row]
        rowname = self._simID(row)
        responses = self._responses(row)
        energy = self.egw[row]
        row = self.waveforms[row]
        output = []
        if not row.incl:
            cosincl = ""
        else:
            cosincl = np.cos(row.incl)

        output.append(self.name)                  # GravEn_SimID
        output.append(strains[0])                 # SimHrss
        output.append(energy)                     # SimEgwR2
        output.append(strains[0])                 # GravEn_Ampl
        output.append(cosincl)           # Internal_x the cosine of the angle the LOS makes with axis of angular momentum
        output.append(row.phi)                    # Intenal_phi angle between source x-axis and the LOS
        output.append(np.cos(np.pi/2.0 - row.dec)) # cos(External_x) # this needs to be the co-declination
        output.append(row.ra if row.ra < np.pi else row.ra - 2*np.pi)
        # ^ External_phi # This is the RA projected onto an Earth-based coordinate system
        output.append(row.psi)                    # External_psi # source's polarisation angle
        output.append(frame.start)                # FrameGPS
        output.append(row.time_geocent_gps)       # EarthCtrGPS
        output.append(rowname)                    # SimName
        output.append(strains[1])                 # SimHpHp
        output.append(strains[2])                 # SimHcHc
        output.append(strains[3])                 # SimHpHp
        output.append(" ".join(" ".join(map(str,l)) for l in responses))
        return ' '.join(str(e) for e in output)

class Frame():
    """
    Represents a frame, in order to prepare the injection frames
    """
    def __init__(self, start, duration, ifo, number = -1):
        """

        Parameters
        ----------
        number : int
           The frame's number within the project. Defaults to -1.
        """
        self.start = start
        self.duration = duration
        self.end = self.start + duration
        self.ifos = ifo
        self.number = -1

    def __repr__(self):
        out = ''
        out += "MDC Frame \n"
        for ifo in self.ifos:
            out += "{} {} {} \n".format(ifo, self.start, self.duration)
        return out

    def get_rowlist(self,mdcs):
        """
        Return the rows from an MDC set which correspond to this frame.

        Parameters
        ----------
        mdcs : MDCSet object
            The set of MDCs from which the rows are to be found.
        """
        return np.where((mdcs.times<self.end)&(mdcs.times>self.start))[0]

    def calculate_n_injections(self, mdcs):
        return len(mdcs.times[(mdcs.times<self.end)&(mdcs.times>self.start)])

    def generate_log(self,mdc):
        log = '#  GravEn_SimID  SimHrss  SimEgwR2  GravEn_Ampl  Internal_x  Internal_phi  External_x  External_phi External_psi  FrameGPS  EarthCtrGPS  SimName  SimHpHp  SimHcHc  SimHpHc  H1       H1ctrGPS        H1fPlus        H1fCross    L1       L1ctrGPS        L1fPlus        L1fCross\n'
        rowlist = self.get_rowlist(mdc)
        for row in rowlist:
            log += mdc.gravEn_row(row, self)
            log += "\n"
        return log

    def generate_gwf(self, mdc, directory, project = "Minke", channel="SCIENCE", force=False, rate=16384.0):
        """
        Produce the gwf file which corresponds to the MDC set over the period of this frame.

        Parameters
        ----------
        mdc : MDCSet object
           The MDC set which should be used to produce this frame.
        directory : str
           The root directory where all of the frames are to be stored, for example
           "/home/albert.einstein/data/mdc/frames/"
           would cause the SineGaussian injections to be made in the directories under
           "/home/albert.einstein/data/mdc/frames/sg"
        project : str
           The name of the project which this frame is a part of. Defaults to 'Minke'.
        channel : str
           The name of the channel which the injections should be made into. This is prepended by the initials
           for each interferometer, so there will be a channel for each interferometer in the gwf.
        force : bool
           If true this forces the recreation of a GWF file even if it already exists.

        Outputs
        -------
        gwf
           The GWF file for this frame.
        """
        ifosstr = "".join(set(ifo[0] for ifo in self.ifos))
        family = mdc.waveforms[0].waveform
        epoch = lal.LIGOTimeGPS(self.start)
        filename = "{}-{}-{}-{}.gwf".format(ifosstr, family, self.start, self.duration)

        self.frame = lalframe.FrameNew(epoch = epoch,
                                       duration = self.duration, project='', run=1, frnum=1,
                                       detectorFlags=lal.LALDETECTORTYPE_ABSENT)

        ifobits = np.array([getattr(lal,"{}_DETECTOR_BIT".format(lal.cached_detector_by_prefix[ifo].frDetector.name.upper()))
                   for ifo in self.ifos])
        ifoflag = numpy.bitwise_or.reduce(ifobits)
        RUN_NUM = -1 # Simulated data should have a negative run number

        head_date = str(self.start)[:5]
        frameloc = directory+"/"+mdc.directory_path()+"/"+head_date+"/"
        mkdir(frameloc)
        if not os.path.isfile(frameloc + filename) or force:
            epoch = lal.LIGOTimeGPS(self.start)
            frame = lalframe.FrameNew(epoch, self.duration, project, RUN_NUM, self.number, ifoflag)
            data = []
            # Loop through each interferometer

            for ifo in self.ifos:
                # Calculate the number of samples in the timeseries
                nsamp = int((self.end-self.start)*rate)
                # Make the timeseries
                h_resp = lal.CreateREAL8TimeSeries("{}:{}".format(ifo, channel), epoch, 0, 1.0/rate, lal.StrainUnit, nsamp)
                # Loop over all of the injections corresponding to this frame
                rowlist = self.get_rowlist(mdc)

                if len(rowlist)==0: return
                for row in rowlist:
                    sim_burst = mdc.waveforms[row]._row()

                    if sim_burst.hrss > 1:
                        distance = sim_burst.amplitude
                    else:
                        distance = None

                    #hp, hx = lalburst.GenerateSimBurst(sim_burst, 1.0/rate);
                    hp, hx, _, _ = mdc.waveforms[row]._generate(rate=rate, half=True, distance=distance)
                    # Apply detector response
                    det = lalsimulation.DetectorPrefixToLALDetector(ifo)
                    # Produce the total strains

                    h_tot = lalsimulation.SimDetectorStrainREAL8TimeSeries(hp, hx,
                                                                           sim_burst.ra, sim_burst.dec, sim_burst.psi, det)
                    # Inject the waveform into the overall timeseries
                    lalsimulation.SimAddInjectionREAL8TimeSeries(h_resp, h_tot, None)

                lalframe.FrameAddREAL8TimeSeriesSimData(frame, h_resp)

            # Make the directory in which to store the files
            # if it doesn't exist already
            mkdir(frameloc)

            # Write out the frame file
            lalframe.FrameWrite(frame, frameloc+filename)

class HWInj(Frame):
    """
    Represents a hardware injection frame.

    Injection frames must be an ASCII file of the hoft sampled at
    the antenna sampling rate, appropriately convolved with an
    antenna response function.

    As a result of the simplicity of this specific output format
    we do not need information such as start-time in the file itself,
    however we should have a sensible naming scheme for the ASCII files
    since they will need to be produced as sidecars for an xml file.

    """
    def __init__(self, ifos):
        """We'll need to know the start-time, the duration, and the ifo
        for each which is to be used for hardware injections in order
        to keep consistency with the data in the xml file, and so that the
        appropriate waveform is injected into the appropriate detector.

        Parameters
        ----------
        ifos : list
           The name of the interferometers, e.g. "L1" for the Livingston, LA LIGO detector.

        """
        self.ifos = ifos

    def __repr__(self):
        """
        The printable representation of this object.
        """
        out = ""
        out += "Hardware MDC Frame \n"
        for ifo in self.ifos:
            out += "{} \n".format(ifo)
        return out

    def generate_pcal(self, mdc, directory, force = False, rate=16384):
        """
        Produce the PCAL-ready hardware injection files as an ASCII list
        sampled at the detector's sample rate.

        Parameters
        ----------
        mdc : MDCSet object
           The signal set which should be used to generate the frame.
        directory : str
           The root directory where all of the frames are to be stored, for example
           "/home/albert.einstein/data/mdc/frames/"
           would cause the SineGaussian injections to be made in the directories under
           "/home/albert.einstein/data/mdc/frames/sg"
        force : bool
           If true this forces the regeneration of the file, even if it
           already exists.

        Outputs
        -------
        ascii file
           The ASCII file containing the correctly sampled waveform convolved with
           the antenna pattern.
        """

        family = mdc.waveforms[0].waveform

        frameloc = os.path.join(directory, (mdc.directory_path()))
        #rowlist = self.get_rowlist(mdc)
        # Unlike with a conventional frame, we need to produce a separate file
        # for each IFO.
        for ifo in self.ifos:
            for sim_burst in mdc.waveforms:
                #sim_burst = mdc.waveforms[row]
                # Check if the file exists, or if we're forcing the creation
                filename = "{}_{}_{}.txt".format(family,
                                                 sim_burst.time,
                                                 ifo)
                if not os.path.isfile(frameloc + filename) or force:
                    data = []
                    epoch = lal.LIGOTimeGPS(sim_burst.time)
                    duration = 10
                    nsamp = duration*rate

                    h_tot = sim_burst._generate_for_detector([ifo], sample_rate=rate)

                    data = np.array(h_tot.data.data)
                    np.savetxt(filename, data)

class HWFrameSet():
    def __init__(self, ifos=["H1", "L1"]):
        """
        A collection of hardware injection frames.

        Parameters
        ----------
        frame_list : str
            The filespath of a CSV file containing the list of frames,
            and the parameters required to produce them: the start and
            duration times, and the interferometers they describe.
        """

        self.frames = []
        self.frames = [HWInj(ifos)]
        #self.frames.append(frame)

    def full_frameset(self, mdc, directory, force=False):
        """
        Produce the gwf files which corresponds to the MDC set over the period of the frames in this collection.

        Parameters
        ----------
        mdc : MDCSet object
           The MDC set which should be used to produce this frame.
        directory : str
           The root directory where all of the frames are to be stored, for example
           "/home/albert.einstein/data/mdc/frames/"
           would cause the SineGaussian injections to be made in the directories under
           "/home/albert.einstein/data/mdc/frames/sg"
        force : bool
           If true this forces the recreation of a GWF file even if it already exists.

        Outputs
        -------
        ascii files
           The ASCII files for these hardware injections.
        """
        for frame in self.frames:
            frame.generate_pcal(mdc, directory, force)

class FrameSet():

    def __init__(self, frame_list):
        """
        A collection of frames.

        Parameters
        ----------
        frame_list : str
            The filespath of a CSV file containing the list of frames,
            and the parameters required to produce them: the start and
            duration times, and the interferometers they describe.
        """

        self.frames = []
        self.frame_list = frame_list = pd.read_csv(frame_list)
        for frame in frame_list.iterrows():
            frame = frame[1]
            ifos = frame['ifo'].replace("['",'').replace("']",'').replace("'",'').split(' ')
            frame = Frame(frame['start time'],frame['duration'],ifos)
            self.frames.append(frame)

    def full_frameset(self, mdc, directory, channel="SCIENCE", force=False):
        """
        Produce the gwf files which corresponds to the MDC set over the period of the frames in this collection.

        Parameters
        ----------
        mdc : MDCSet object
           The MDC set which should be used to produce this frame.
        directory : str
           The root directory where all of the frames are to be stored, for example
           "/home/albert.einstein/data/mdc/frames/"
           would cause the SineGaussian injections to be made in the directories under
           "/home/albert.einstein/data/mdc/frames/sg"
        channel : str
           The name of the channel which the injections should be made into. This is prepended by the initials
           for each interferometer, so there will be a channel for each interferometer in the gwf.
        force : bool
           If true this forces the recreation of a GWF file even if it already exists.

        Outputs
        -------
        gwf files
           The GWF files for these frames.
        """
        for frame in self.frames:
            frame.generate_gwf(mdc, directory, channel, force)

    def full_logfile(self, mdc, location):
        """
        Produce a log file for the entire frame set
        """
        full_log = ''
        for frame in self.frames:
            full_log += frame.generate_log(mdc)

        with open(location, "w") as text_file:
            text_file.write(full_log)


"""
Implementations of various common operations, like `show()` for displaying an
array or with matplotlib, and `stats()` for computing min/max/avg.  Most can
handle a numpy array or `rasterio.Band()`.  Primarily supports `$ rio insp`.
"""

from __future__ import absolute_import

import code
import collections
import logging
import warnings

try:
    import matplotlib.pyplot as plt
except ImportError:
    plt = None
except RuntimeError as e:
    # Certain environment configurations can trigger a RuntimeError like:

    # Trying to import matplotlibRuntimeError: Python is not installed as a
    # framework. The Mac OS X backend will not be able to function correctly
    # if Python is not installed as a framework. See the Python ...
    warnings.warn(str(e), RuntimeWarning, stacklevel=2)
    plt = None

import numpy

import rasterio
from rasterio.five import zip_longest

logger = logging.getLogger('rasterio')

Stats = collections.namedtuple('Stats', ['min', 'max', 'mean'])

# Collect dictionary of functions for use in the interpreter in main()
funcs = locals()

def show(source, cmap='gray', with_bounds=True):
    """
    Display a raster or raster band using matplotlib.

    Parameters
    ----------
    source : array-like or (raster dataset, bidx)
        If array-like, should be of format compatible with
        matplotlib.pyplot.imshow. If the tuple (raster dataset, bidx),
        selects band `bidx` from raster.
    cmap : str (opt)
        Specifies the colormap to use in plotting. See
        matplotlib.Colors.Colormap. Default is 'gray'.
    with_bounds : bool (opt)
        Whether to change the image extent to the spatial bounds of the image,
        rather than pixel coordinates. Only works when source is
        (raster dataset, bidx).
    """

    if isinstance(source, tuple):
        arr = source[0].read(source[1])
        xs = source[0].res[0] / 2.
        ys = source[0].res[1] / 2.
        if with_bounds:
            extent = (source[0].bounds.left - xs, source[0].bounds.right - xs,
                      source[0].bounds.bottom - ys, source[0].bounds.top - ys)
        else:
            extent = None
    else:
        arr = source
        extent = None
    if plt is not None:
        imax = plt.imshow(arr, cmap=cmap, extent=extent)
        fig = plt.gcf()
        fig.show()
    else:
        raise ImportError("matplotlib could not be imported")

def stats(source):
    """Return a tuple with raster min, max, and mean.
    """
    if isinstance(source, tuple):
        arr = source[0].read(source[1])
    else:
        arr = source
    return Stats(numpy.min(arr), numpy.max(arr), numpy.mean(arr))

def show_hist(source, bins=10, masked=True, title='Histogram'):

    """
    Easily display a histogram with matplotlib.

    Parameters
    ----------
    bins : int, optional
        Compute histogram across N bins.
    data : np.array or rasterio.Band or tuple(dataset, bidx)
        Input data to display.  The first three arrays in multi-dimensional
        arrays are plotted as red, green, and blue.
    masked : bool, optional
        When working with a `rasterio.Band()` object, specifies if the data
        should be masked on read.
    title : str, optional
        Title for the figure.
    """

    if plt is None:
        raise ImportError("Could not import matplotlib")

    if isinstance(source, (tuple, rasterio.Band)):
        arr = source[0].read(source[1], masked=masked)
    else:
        arr = source

    # The histogram is computed individually for each 'band' in the array
    # so we need the overall min/max to constrain the plot
    rng = arr.min(), arr.max()

    if len(arr.shape) is 2:
        arr = [arr]
        colors = ['gold']
    else:
        colors = ('red', 'green', 'blue', 'violet', 'gold', 'saddlebrown')

    # If a rasterio.Band() is given make sure the proper index is displayed
    # in the legend.
    if isinstance(source, (tuple, rasterio.Band)):
        labels = [str(source[1])]
    else:
        labels = (str(i + 1) for i in range(len(arr)))

    # This loop should add a single plot each band in the input array,
    # regardless of if the number of bands exceeds the number of colors.
    # The colors slicing ensures that the number of iterations always
    # matches the number of bands.
    # The goal is to provide a curated set of colors for working with
    # smaller datasets and let matplotlib define additional colors when
    # working with larger datasets.
    for bnd, color, label in zip_longest(arr, colors[:len(arr)], labels):

        plt.hist(
            bnd.flatten(),
            bins=bins,
            alpha=0.5,
            color=color,
            label=label,
            range=rng
        )

    plt.legend(loc="upper right")
    plt.title(title, fontweight='bold')
    plt.grid(True)
    plt.xlabel('DN')
    plt.ylabel('Frequency')
    fig = plt.gcf()
    fig.show()

def main(banner, dataset, alt_interpreter=None):
    """ Main entry point for use with python interpreter """
    local = dict(funcs, src=dataset, np=numpy, rio=rasterio, plt=plt)
    if not alt_interpreter:
        code.interact(banner, local=local)
    elif alt_interpreter == 'ipython':
        import IPython
        IPython.InteractiveShell.banner1 = banner
        IPython.start_ipython(argv=[], user_ns=local)
    else:
        raise ValueError("Unsupported interpreter '%s'" % alt_interpreter)

    return 0


from __future__ import division
import numpy as np
import math as m
from easygui import multenterbox
import pandas as pd
import matplotlib.pyplot as plt
import math as m

def import_xl(file_path):
    df = pd.read_excel(file_path,header = None)
    df = df.values
    return df

def export_xl(file_path,sheets):
    writer = pd.ExcelWriter(file_path)
    for sheet,name in sheets.items():
        df = pd.DataFrame(name)
        df.to_excel(writer,sheet)
    writer.save()

#Henter ut en kolonne
def column(matrix, i):
    return [row[i] for row in matrix]

#Henter ut en rad
def row(matrix, i):
    return [column[i] for column in matrix]

#Selection sort O(n2)
def selection_sort(array):
    n = len(array)
    for i in range(0,n):
        smallest = i
        for j in range(i,n):
            if array[j]<array[smallest]:
                smallest = j
        copy = array[i]
        array[i] = array[smallest]
        array[smallest] = copy
    return array

#Leser om to lister inneholder minst ett felles tall
def common_node(array_1,array_2):
    x = selection_sort(array_1)
    y = selection_sort(array_2)
    i = 0
    j = 0
    share = 0
    stop = max([len(x),len(y)])-1
    while min([i,j])< stop:
        if x[i]>y[j]:
            j+=1
        elif x[i]<y[j]:
            i+=1
        else:
            share = 1
            j = 10**6
            i = 10**6
    return share

def common_node_count(array_1,array_2):
    x = selection_sort(array_1)
    y = selection_sort(array_2)
    i = 0
    j = 0
    share = 0
    while i < len(x) and j < len(y):
        if x[i]>y[j]:
            j+=1
        elif x[i]<y[j]:
            i+=1
        else:
            share += 1
            j +=1
            i +=1
    return share

#KORTERSTE RUTE FUNKSJONER

#Lager en graf fra lenke-liste
def make_graph(array):
    #nodes = common_node_count(column(array,0),column(array,1))
    nodes = 35
    matrix = np.full((nodes,nodes),10**6) #Lager matrise med store tall som byttes
    for i in range(0,len(array)): #Hovedloop
        #Trekker fra en for sammenlignbarhet med python-arrays
        matrix[array[i][1]-1][array[i][0]-1] = array[i][2]
        matrix[array[i][0]-1][array[i][1]-1] = array[i][2]
    np.fill_diagonal(matrix, 0)
    return matrix

#Lager lengdematrise n x n
def floyd_warshall(array):
    matrix = make_graph(array)
    #nodes = common_node_count(column(array,0),column(array,1))
    nodes = 35
    pred = np.full((nodes,nodes),-1)

    for i in range(0,nodes):
        for j in range(0,nodes):
            if i != j:
                pred[i][j] = i

    for k in range(0,nodes):
        for i in range(0,nodes):
            for j in range(0,nodes):
                if matrix[i][j] > matrix[i][k] +  matrix[k][j]:
                    matrix[i][j] = matrix[i][k] +  matrix[k][j]
                    pred[i][j] = pred[k][j]
    return matrix,pred

#Laster inn data fra en csv fil til et nettverksarray
def get_network(net_csv):
    graf = open(net_csv,'r')
    lenker=0
    for line in graf:
        lenker+=1
    graf_edit = np.full((lenker, 3),0)
    graf = open(net_csv,'r')
    k = 0
    for line in graf:
        stuff = line.split(";")
        graf_edit[k][0] = float(stuff[0])
        graf_edit[k][1] = float(stuff[1])
        temp = stuff[2].split('\n')[0]
        graf_edit[k][2] = float(temp)
        k+=1
    return graf_edit

#Lager en path-vektor
def path(p,i,j,path_vec):
    if i == j:
        path_vec.append(i)
    else:
        path(p, i, p[i][j],path_vec)
        path_vec.append(j)

#Henter en spesifikk path
def get_path(p,i,j):
    #j = j + 1
    path_vec=[]
    path(p,i,j,path_vec)
    #for i in range(1,len(path_vec)):
    #    path_vec[i] = path_vec[i] - 1
    return path_vec

#Lager adjecency-matrise (ikke ferdig)
def build_adj(pred):

    adj_mat = np.zeros((len(pred),len(pred)))
    array_a = []
    array_b = []
    for i in range(1,len(pred)):
        for j in range(1,len(pred)):
            array_a = get_path(pred,i,j)
            print array_a
            array_b = get_path(pred,2,10)
            print array_b
            try:
                adj_mat[1][j] = common_node(array_a,array_b)
            except:
                adj_mat[1][j] = 0

            print adj_mat[1][j]

    return adj_mat

#Nettverkslaster
#Argumenter: (1) Forgjenger-matrise (2) antall noder (3) nettverksfil (4) od-matrise
def network_loader(graf,net,od,pred):

    #Antall noder
    n = len(od)-1

    #Redigering
    for k in range(0,len(net)):
        net[k][3]=0 #Nulllstiller antall reiser
        net[k][2]=graf[k][2] #Legger inn oppdaterteavstander fra grafen

    #Legger ut reiser paa nettet
    for i in range(0,n):
        for j in range(0,n):
            path = get_path(pred,i,j)
            len_path=get_len_path(path)
            for h in range(0,len_path):
                for k in range(0,len(net)):
                    if net[k][0] == path[h]+1 and net[k][1] == path[1+h]+1:
                        net[k][3] += int(od[i][j])
                    elif net[k][1] == path[h]+1 and net[k][0] == path[1+h]+1:
                        net[k][3] += int(od[i][j])
    return net

#a=get_path(pred,5,12)

#GRAVITASJONSFUNKSJONER
def deter_mat_make(length_mat):

    deter_mat = np.zeros((len(length_mat),len(length_mat)))
    for i in range(0,len(length_mat)):
        for j in range(0,len(length_mat)):
            deter_mat[i][j] = deter(length_mat[i][j])
    return deter_mat

def deter(length):
    return 2.71**(beta*length)

def sumproduct(list1,list2):

    sums = 0
    for i in range(0,len(list1)):
        sums += list1[i]*list2[i]
    return sums

def gravity(origin, destination, length_mat):

    #Initialization
    deter_mat = deter_mat_make(length_mat) #Lager matrise med forvitring
    dimension = len(origin) #Henter ut matrisedimensjonene
    alpha = [1]*(dimension) #Intitierer alpha-vektor
    beta = [1]*(dimension)  #Intitierer beta-vektor
    largest = 10**6         #Intitierer storste avvik
    alpha_last = alpha      #Intitierer alpha -1
    beta_last = beta        #Intitierer beta -1
    k = 0                   #Intitierer tellevariabler for iterasjoner
    iterasjoner = []

    #Hovedlokke
    while largest > .00001:

        #Oppdaterer faktorene
       for p in range(0,dimension):
          alpha[p] = origin[p]/(sumproduct(beta_last,column(deter_mat,p)))
          beta[p] = destination[p]/(sumproduct(alpha,row(deter_mat,p)))
          largest = 0

        #Looper for aa finne storste element
       for j in range(0,dimension):
           current = alpha[j]*sumproduct(beta,column(deter_mat,j))-origin[j]
           if current>largest:
               largest = current

        #Setter forrige beta
       beta_last = beta
       iterasjoner.append(largest)
       #Legger til en iterasjon
       k+=1
       print "Konvergens, Gravitasjonsmodell", largest
       if k == maxiter:
           largest = 0

    return alpha,beta,k,iterasjoner

def create_od(origin,destination, length_mat):

    alpha,beta,k,iterasjoner = gravity(origin, destination, length_mat)
    deter_mat = deter_mat_make(length_mat)
    od = np.zeros((len(origin),len(origin)))

    for i in range(0,len(origin)):
        for j in range(0,len(origin)):
            od[i][j] = alpha[i]*beta[j]*deter_mat[i][j]
    return od,alpha,beta,k,iterasjoner

def calc_pt_matrix(od,length_mat):
    out_od = np.zeros((len(od),len(od)))
    for i in range(0,len(od)):
        for j in range(0,len(od)):
            out_od[i][j] = int(out_od[i][j])*length_mat[i][j]
    return out_od

def get_min(net):
    smallest = 10**6
    smallest_id = 10**6
    for i in range(0,len(net)):
        if net[i][3]/net[i][2]<smallest and net[i][5]==0:
            smallest = net[i][3]/net[i][2]
            smallest_id = i
    return smallest_id,smallest

def change_graph(graph,net):
    graph_out = graph
    for i in range(0,len(net)):
        if net[i][5]==1:
            graph_out[i][2]=k_just*graph_out[i][2]
    return graph_out

def production(net):
    sumcost = 0
    for i in range(0,len(net)):
        if net[i][5]!=1:
            sumcost += (net[i][3]/capacity)*net[i][2]
    return sumcost

def sum_pass(net):
    sumpass = 0
    for i in range(0,len(net)):
        sumpass+=net[i][3]
    return sumpass

def get_len_path(path):
    len_path = 0
    if len(path) < 3:
        len_path = 0
    elif len(path) == 3:
        len_path = 2
    else:
        len_path=int(len(path)/2)+int(len(path)%2)+1
    return len_path

def obj(od,length_mat,net,prodgoal):
    return (production(net)*kmk*dogn-prodgoal)**2*(k_just-1)*capacity/.9+time_cost(od,length_mat)

def time_cost(od,length_mat):
    cost = 0
    for i in range(0,len(od)-1):
        for j in range(0,len(od)-1):
            cost += od[i][j]*length_mat[i][j]
    return cost

def get_zero_net(net):
    zero_net = np.zeros((len(net),6))
    for i in range(0,len(net)):
        zero_net[i][2] = net[i][2]
        zero_net[i][3] = net[i][3]
        zero_net[i][5] = net[i][5]
    return zero_net

def update_zero_net(net,zero_net):
    for i in range(0,len(net)):
        zero_net[i][5] = net[i][5]
    return zero_net


# Plot Empirical Cumulative Distribution Function for Object Size

import os
import cv2
import pandas as pd
from plantcv.plantcv import params
from plantcv.plantcv._debug import _debug
from statsmodels.distributions.empirical_distribution import ECDF
from plotnine import ggplot, aes, geom_point, labels, scale_x_log10

def obj_size_ecdf(mask, title=None):
    """
    Plot empirical cumulative distribution for object size based on binary mask.

    Inputs:
    mask  = binary mask
    title = a custom title for the plot (default=None)

    Returns:
    fig_ecdf = empirical cumulative distribution function plot

    :param mask: numpy.ndarray
    :param title: str
    :return fig_ecdf: plotnine.ggplot.ggplot
    """
    objects, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)[-2:]
    areas = [cv2.contourArea(cnt) for cnt in objects]
    # Remove objects with areas < 1px
    areas = [i for i in areas if i >= 1.0]

    ecdf = ECDF(areas, side='right')

    ecdf_df = pd.DataFrame({'object area': ecdf.x[1:], 'cumulative probability': ecdf.y[1:]})
    # create ecdf plot and apply log-scale for x-axis (areas)
    fig_ecdf = (ggplot(data=ecdf_df, mapping=aes(x='object area', y='cumulative probability'))
                + geom_point(size=.1)
                + scale_x_log10())
    if title is not None:
        fig_ecdf = fig_ecdf + labels.ggtitle(title)

    # Plot or print the ecdf
    _debug(visual=fig_ecdf,
           filename=os.path.join(params.debug_outdir, str(params.device) + '_area_ecdf.png'))
    return fig_ecdf


"""
TupleBanditsPureExplorationDashboard
author: Nick Glattard, n.glattard@gmail.com
last updated: 4/24/2015

######################################
TupleBanditsPureExplorationDashboard

"""

import json
import numpy
import numpy.random
import matplotlib.pyplot as plt
from datetime import datetime
from datetime import timedelta
from next.utils import utils
from next.apps.AppDashboard import AppDashboard

class TupleBanditsPureExplorationDashboard(AppDashboard):

    def __init__(self,db,ell):
        AppDashboard.__init__(self,db,ell)

    def get_app_supported_stats(self):
        """
        Returns a list of dictionaries describing the identifier (stat_id) and
        necessary params inputs to be used when calling getStats

        Expected output (list of dicts, each with fields):
            (string) stat_id : the identiifer of the statistic
            (string) description : docstring of describing outputs
            (list of string) necessary_params : list where each string describes the type of param input like 'alg_label' or 'task'
        """
        stat_list = self.get_supported_stats()

        stat = {}
        stat['stat_id'] = 'most_current_ranking'
        stat['description'] = self.most_current_ranking.__doc__
        stat['necessary_params'] = ['alg_label']
        stat_list.append(stat)

        return stat_list

    def most_current_ranking(self,app_id,exp_uid,alg_label):
        """
        Description: Returns a ranking of arms in the form of a list of dictionaries, which is conveneint for downstream applications

        Expected input:
          (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

        The 'headers' contains a list of dictionaries corresponding to each column of the table with fields 'label' and 'field' where 'label' is the label of the column to be put on top of the table, and 'field' is the name of the field in 'data' that the column correpsonds to

        Expected output (in dict):
          plot_type : 'columnar_table'
          headers : [ {'label':'Rank','field':'rank'}, {'label':'Target','field':'index'} ]
          (list of dicts with fields) data (each dict is a row, each field is the column for that row):
            (int) index : index of target
            (int) ranking : rank (0 to number of targets - 1) representing belief of being best arm
        """

        alg_list,didSucceed,message = self.db.get(app_id+':experiments',exp_uid,'alg_list')

        for algorithm in alg_list:
            if algorithm['alg_label'] == alg_label:
                alg_id = algorithm['alg_id']
                alg_uid = algorithm['alg_uid']

        list_of_log_dict,didSucceed,message = self.ell.get_logs_with_filter(app_id+':ALG-EVALUATION',{'alg_uid':alg_uid})
        list_of_log_dict = sorted(list_of_log_dict, key=lambda k: k['num_reported_answers'] )

        print didSucceed, message

        item = list_of_log_dict[-1]

        return_dict = {}
        return_dict['headers'] = [{'label':'Rank','field':'rank'},{'label':'Target','field':'index'},{'label':'Score','field':'score'},{'label':'Precision','field':'precision'}]
        return_dict['data'] = item['targets']
        return_dict['plot_type'] = 'columnar_table'

        return return_dict


#!/usr/bin/env python

import os, time
import numpy as np
import matplotlib.pyplot as plt
import scipy.signal

results_directory = os.getenv('HOME') + "/rfi_capture_results/"
SAMPLE_FREQUENCY = 3600.0 # MHz and us
ADC_SCALE_VALUE = 707.94

# algorithm:
# open a .npy file (or do the disk buffer thing)

filename = raw_input("what file should be open? [most recent]  ")
if filename == "": # default to the most recent file
    filename = "/tmp/rfi_signal.npy"
else:
    filename = results_directory + filename
signal = np.load(filename)
decimation_factor = int(len(signal)/2**20) + 1
print "decimation factor: " + str(decimation_factor)
if decimation_factor >= 2 :
    signal_decimated = scipy.signal.decimate(signal, decimation_factor, n=1, ftype="fir")
else:
    signal_decimated = signal
print "len : " + str(len(signal_decimated))
axis = np.linspace(0, decimation_factor * len(signal_decimated)/SAMPLE_FREQUENCY, len(signal_decimated), endpoint=False)
plt.plot(axis, signal_decimated, "b.")
plt.show()
# plot the signal decimated by a paramamter (defualt: 1)

# ask the user to input a subplot time
start_time = float(raw_input("At what time (microseconds) does the signal start?  "))
end_time = float(raw_input("At what time (microseconds) does the signal end?   "))
start_sample = int( start_time * SAMPLE_FREQUENCY )
end_sample = int( end_time * SAMPLE_FREQUENCY )

subsignal = signal[start_sample:end_sample]
subsignal_axis = np.linspace(start_time, end_time, len(subsignal), endpoint=False)
spectrum = np.fft.rfft(subsignal)
spectrum_axis = np.linspace(0, SAMPLE_FREQUENCY/2, len(spectrum), endpoint=False)

plt.subplot(211)
plt.plot(subsignal_axis, subsignal)
plt.subplot(212)
plt.plot(spectrum_axis, 10*np.log10( np.abs(spectrum) / (ADC_SCALE_VALUE*len(spectrum) )))
plt.show()

# plot the subplot and the fft of the subplot


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

from keras.layers import Dense
from keras.preprocessing.image import apply_transform

import matplotlib.pyplot as plt

def take_glimpses(image, location, sizes):
    glimpses = []

    resize = sizes[0]
    for size in sizes:
        glimpse = tf.image.extract_glimpse(image, size=size, offsets=location,
                                           normalized=True, centered=True, uniform_noise=False)
        glimpses += [tf.image.resize_images(glimpse, resize)]

    return glimpses

def glimpse_network(image, location, sizes, activation="relu",
                    glimpse_num_features=128, location_num_features=128, output_dim=256):
    assert len(sizes) == 3

    with tf.variable_scope("glimpse_network"):
        glimpses = []

        resize = sizes[0]
        for size in sizes:
            glimpse = tf.image.extract_glimpse(image, size=size, offsets=location, uniform_noise=False,
                                               normalized=True, centered=True)
            glimpses += [tf.image.resize_images(glimpse, resize[0], resize[1])]

        glimpse = tf.concat(-1, glimpses)
        glimpse = tf.reshape(glimpse, (-1, np.prod(resize) * len(sizes)))
        glimpse_feature = Dense(glimpse_num_features, activation=activation)(glimpse)
        location_feature = Dense(location_num_features, activation=activation)(location)

        feature = Dense(output_dim, activation=activation)(glimpse_feature + location_feature)
        return feature, glimpses

def accuracy_score(y_preds, y_true):
    return np.sum((y_preds == y_true).astype(np.float32)) / y_preds.shape[0]

def translate(batch_x, size=(128, 128)):
    """Make translated mnist"""
    height = batch_x.shape[1]
    width = batch_x.shape[2]
    X = np.zeros((batch_x.shape[0],) + size + (1,), dtype=batch_x.dtype)
    X[:, :height, :width, :] = batch_x
    for i, x in enumerate(X[:]):
        tx = np.random.uniform(-(size[1] - width), 0)
        ty = np.random.uniform(-(size[0] - height), 0)

        translation_matrix = np.asarray([
            [1, 0, tx],
            [0, 1, ty],
            [0, 0, 1]
        ], dtype=batch_x.dtype)

        X[i, :, :, :] = apply_transform(x, translation_matrix, channel_index=2, fill_mode="nearest", cval=0.)
    return X

def plot_glimpse(images, locations, name="glimpse.png"):
    image = images[0]
    location = locations[:, 0, :]

    fig = plt.figure()
    plt.imshow(image, cmap=plt.get_cmap("gray"))
    plt.plot(location[:, 0], location[:, 1])
    for i, (x, y) in enumerate(location):
        plt.annotate("t=%d" % i, xy=(x, y), xytext=(-10, 10),
                     textcoords="offset points", ha="right", va="bottom",
                     bbox=dict(boxstyle="round, pad=0.5", fc="white", alpha=0.5),
                     arrowprops=dict(arrowstyle="->", connectionstyle="arc3, rad=0"))
    plt.savefig(name)
    plt.gcf().clear()
    plt.close("all")


# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
# License: BSD 3 clause

import numpy as np
from scipy import sparse

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises

from sklearn.linear_model.randomized_l1 import (lasso_stability_path,
                                                RandomizedLasso,
                                                RandomizedLogisticRegression)
from sklearn.datasets import load_diabetes, load_iris
from sklearn.feature_selection import f_regression, f_classif
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model.base import center_data

diabetes = load_diabetes()
X = diabetes.data
y = diabetes.target
X = StandardScaler().fit_transform(X)
X = X[:, [2, 3, 6, 7, 8]]

# test that the feature score of the best features
F, _ = f_regression(X, y)

def test_lasso_stability_path():
    """Check lasso stability path"""
    # Load diabetes data and add noisy features
    scaling = 0.3
    coef_grid, scores_path = lasso_stability_path(X, y, scaling=scaling,
                                                  random_state=42,
                                                  n_resampling=30)

    assert_array_equal(np.argsort(F)[-3:],
                       np.argsort(np.sum(scores_path, axis=1))[-3:])

def test_randomized_lasso():
    """Check randomized lasso"""
    scaling = 0.3
    selection_threshold = 0.5

    # or with 1 alpha
    clf = RandomizedLasso(verbose=False, alpha=1, random_state=42,
                          scaling=scaling,
                          selection_threshold=selection_threshold)
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:])

    # or with many alphas
    clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42,
                          scaling=scaling,
                          selection_threshold=selection_threshold)
    feature_scores = clf.fit(X, y).scores_
    assert_equal(clf.all_scores_.shape, (X.shape[1], 2))
    assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:])

    X_r = clf.transform(X)
    X_full = clf.inverse_transform(X_r)
    assert_equal(X_r.shape[1], np.sum(feature_scores > selection_threshold))
    assert_equal(X_full.shape, X.shape)

    clf = RandomizedLasso(verbose=False, alpha='aic', random_state=42,
                          scaling=scaling)
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(feature_scores, X.shape[1] * [1.])

    clf = RandomizedLasso(verbose=False, scaling=-0.1)
    assert_raises(ValueError, clf.fit, X, y)

    clf = RandomizedLasso(verbose=False, scaling=1.1)
    assert_raises(ValueError, clf.fit, X, y)

def test_randomized_logistic():
    """Check randomized sparse logistic regression"""
    iris = load_iris()
    X = iris.data[:, [0, 2]]
    y = iris.target
    X = X[y != 2]
    y = y[y != 2]

    F, _ = f_classif(X, y)

    scaling = 0.3
    clf = RandomizedLogisticRegression(verbose=False, C=1., random_state=42,
                                       scaling=scaling, n_resampling=50,
                                       tol=1e-3)
    X_orig = X.copy()
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(X, X_orig)   # fit does not modify X
    assert_array_equal(np.argsort(F), np.argsort(feature_scores))

    clf = RandomizedLogisticRegression(verbose=False, C=[1., 0.5],
                                       random_state=42, scaling=scaling,
                                       n_resampling=50, tol=1e-3)
    feature_scores = clf.fit(X, y).scores_
    assert_array_equal(np.argsort(F), np.argsort(feature_scores))

def test_randomized_logistic_sparse():
    """Check randomized sparse logistic regression on sparse data"""
    iris = load_iris()
    X = iris.data[:, [0, 2]]
    y = iris.target
    X = X[y != 2]
    y = y[y != 2]

    # center here because sparse matrices are usually not centered
    X, y, _, _, _ = center_data(X, y, True, True)

    X_sp = sparse.csr_matrix(X)

    F, _ = f_classif(X, y)

    scaling = 0.3
    clf = RandomizedLogisticRegression(verbose=False, C=1., random_state=42,
                                       scaling=scaling, n_resampling=50,
                                       tol=1e-3)
    feature_scores = clf.fit(X, y).scores_
    clf = RandomizedLogisticRegression(verbose=False, C=1., random_state=42,
                                       scaling=scaling, n_resampling=50,
                                       tol=1e-3)
    feature_scores_sp = clf.fit(X_sp, y).scores_
    assert_array_equal(feature_scores, feature_scores_sp)


"""
Example code showing how to draw an interactive matplotlib figure
in Excel.

While the figure is displayed Excel is still useable in the background
and the chart may be updated with new data by calling the same
function again.
"""
from pyxll import xl_func
from pandas.stats.moments import ewma

# matplotlib imports
from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure

# Qt imports
from PySide import QtCore, QtGui
import timer  # for polling the Qt application

# dict to keep track of any chart windows
_plot_windows = {}

@xl_func("string figname, numpy_column<float> xs, numpy_column<float> ys, int span: string")
def mpl_plot_ewma(figname, xs, ys, span):
    """
    Show a matplotlib line plot of xs vs ys and ewma(ys, span) in an interactive window.

    :param figname: name to use for this plot's window
    :param xs: list of x values as a column
    :param ys: list of y values as a column
    :param span: ewma span
    """
    # Get the Qt app.
    # Note: no need to 'exec' this as it will be polled in the main windows loop.
    app = get_qt_app()

    # create the figure and axes for the plot
    fig = Figure(figsize=(600, 600), dpi=72, facecolor=(1, 1, 1), edgecolor=(0, 0, 0))
    ax = fig.add_subplot(111)

    # calculate the moving average
    ewma_ys = ewma(ys, span=span)

    # plot the data
    ax.plot(xs, ys, alpha=0.4, label="Raw")
    ax.plot(xs, ewma_ys, label="EWMA")
    ax.legend()

    # generate the canvas to display the plot
    canvas = FigureCanvas(fig)

    # Get or create the Qt windows to show the chart in.
    if figname in _plot_windows:
        # get from the global dict and clear any previous widgets
        window = _plot_windows[figname]
        layout = window.layout()
        if layout:
            for i in reversed(range(layout.count())):
                layout.itemAt(i).widget().setParent(None)
    else:
        # create a new window for this plot and store it for next time
        window = QtGui.QWidget()
        window.resize(800, 600)
        window.setWindowTitle(figname)
        _plot_windows[figname] = window

    # create the navigation toolbar
    toolbar = NavigationToolbar(canvas, window)

    # add the canvas and toolbar to the window
    layout = window.layout() or QtGui.QVBoxLayout()
    layout.addWidget(canvas)
    layout.addWidget(toolbar)
    window.setLayout(layout)

    window.show()

    return "[Plotted '%s']" % figname

#
# Taken from the ui/qt.py example
#
def get_qt_app():
    """
    returns the global QtGui.QApplication instance and starts
    the event loop if necessary.
    """
    app = QtCore.QCoreApplication.instance()
    if app is None:
        # create a new application
        app = QtGui.QApplication([])

        # use timer to process events periodically
        processing_events = {}
        def qt_timer_callback(timer_id, time):
            if timer_id in processing_events:
                return
            processing_events[timer_id] = True
            try:
                app = QtCore.QCoreApplication.instance()
                if app is not None:
                    app.processEvents(QtCore.QEventLoop.AllEvents, 300)
            finally:
                del processing_events[timer_id]

        timer.set_timer(100, qt_timer_callback)

    return app


"""Nearest Neighbors graph functions"""

# Author: Jake Vanderplas <vanderplas@astro.washington.edu>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam

import warnings

from .base import KNeighborsMixin, RadiusNeighborsMixin
from .unsupervised import NearestNeighbors

def _check_params(X, metric, p, metric_params):
    """Check the validity of the input parameters"""
    params = zip(['metric', 'p', 'metric_params'],
                 [metric, p, metric_params])
    est_params = X.get_params()
    for param_name, func_param in params:
        if func_param != est_params[param_name]:
            raise ValueError(
                "Got %s for %s, while the estimator has %s for "
                "the same parameter." % (
                    func_param, param_name, est_params[param_name]))

def _query_include_self(X, include_self, mode):
    """Return the query based on include_self param"""
    # Done to preserve backward compatibility.
    if include_self is None:
        if mode == "connectivity":
            warnings.warn(
                "The behavior of 'kneighbors_graph' when mode='connectivity' "
                "will change in version 0.18. Presently, the nearest neighbor "
                "of each sample is the sample itself. Beginning in version "
                "0.18, the default behavior will be to exclude each sample "
                "from being its own nearest neighbor. To maintain the current "
                "behavior, set include_self=True.", DeprecationWarning)
            include_self = True
        else:
            include_self = False

    if include_self:
        query = X._fit_X
    else:
        query = None

    return query

def kneighbors_graph(X, n_neighbors, mode='connectivity', metric='minkowski',
                     p=2, metric_params=None, include_self=None):
    """Computes the (weighted) graph of k-Neighbors for points in X

    Read more in the :ref:`User Guide <unsupervised_neighbors>`.

    Parameters
    ----------
    X : array-like or BallTree, shape = [n_samples, n_features]
        Sample data, in the form of a numpy array or a precomputed
        :class:`BallTree`.

    n_neighbors : int
        Number of neighbors for each sample.

    mode : {'connectivity', 'distance'}, optional
        Type of returned matrix: 'connectivity' will return the
        connectivity matrix with ones and zeros, in 'distance' the
        edges are Euclidean distance between points.

    metric : string, default 'minkowski'
        The distance metric used to calculate the k-Neighbors for each sample
        point. The DistanceMetric class gives a list of available metrics.
        The default distance is 'euclidean' ('minkowski' metric with the p
        param equal to 2.)

    include_self: bool, default backward-compatible.
        Whether or not to mark each sample as the first nearest neighbor to
        itself. If `None`, then True is used for mode='connectivity' and False
        for mode='distance' as this will preserve backwards compatibilty. From
        version 0.18, the default value will be False, irrespective of the
        value of `mode`.

    p : int, default 2
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params: dict, optional
        additional keyword arguments for the metric function.

    Returns
    -------
    A : sparse matrix in CSR format, shape = [n_samples, n_samples]
        A[i, j] is assigned the weight of edge that connects i to j.

    Examples
    --------
    >>> X = [[0], [3], [1]]
    >>> from sklearn.neighbors import kneighbors_graph
    >>> A = kneighbors_graph(X, 2)
    >>> A.toarray()
    array([[ 1.,  0.,  1.],
           [ 0.,  1.,  1.],
           [ 1.,  0.,  1.]])

    See also
    --------
    radius_neighbors_graph
    """
    if not isinstance(X, KNeighborsMixin):
        X = NearestNeighbors(n_neighbors, metric=metric, p=p,
                             metric_params=metric_params).fit(X)
    else:
        _check_params(X, metric, p, metric_params)

    query = _query_include_self(X, include_self, mode)
    return X.kneighbors_graph(X=query, n_neighbors=n_neighbors, mode=mode)

def radius_neighbors_graph(X, radius, mode='connectivity', metric='minkowski',
                           p=2, metric_params=None, include_self=None):
    """Computes the (weighted) graph of Neighbors for points in X

    Neighborhoods are restricted the points at a distance lower than
    radius.

    Read more in the :ref:`User Guide <unsupervised_neighbors>`.

    Parameters
    ----------
    X : array-like or BallTree, shape = [n_samples, n_features]
        Sample data, in the form of a numpy array or a precomputed
        :class:`BallTree`.

    radius : float
        Radius of neighborhoods.

    mode : {'connectivity', 'distance'}, optional
        Type of returned matrix: 'connectivity' will return the
        connectivity matrix with ones and zeros, in 'distance' the
        edges are Euclidean distance between points.

    metric : string, default 'minkowski'
        The distance metric used to calculate the neighbors within a
        given radius for each sample point. The DistanceMetric class
        gives a list of available metrics. The default distance is
        'euclidean' ('minkowski' metric with the param equal to 2.)

    include_self: bool, default None
        Whether or not to mark each sample as the first nearest neighbor to
        itself. If `None`, then True is used for mode='connectivity' and False
        for mode='distance' as this will preserve backwards compatibilty. From
        version 0.18, the default value will be False, irrespective of the
        value of `mode`.

    p : int, default 2
        Power parameter for the Minkowski metric. When p = 1, this is
        equivalent to using manhattan_distance (l1), and euclidean_distance
        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.

    metric_params: dict, optional
        additional keyword arguments for the metric function.

    Returns
    -------
    A : sparse matrix in CSR format, shape = [n_samples, n_samples]
        A[i, j] is assigned the weight of edge that connects i to j.

    Examples
    --------
    >>> X = [[0], [3], [1]]
    >>> from sklearn.neighbors import radius_neighbors_graph
    >>> A = radius_neighbors_graph(X, 1.5)
    >>> A.toarray()
    array([[ 1.,  0.,  1.],
           [ 0.,  1.,  0.],
           [ 1.,  0.,  1.]])

    See also
    --------
    kneighbors_graph
    """
    if not isinstance(X, RadiusNeighborsMixin):
        X = NearestNeighbors(radius=radius, metric=metric, p=p,
                             metric_params=metric_params).fit(X)
    else:
        _check_params(X, metric, p, metric_params)

    query = _query_include_self(X, include_self, mode)
    return X.radius_neighbors_graph(query, radius, mode)


# Author: Lars Buitinck <L.J.Buitinck@uva.nl>
# License: 3-clause BSD

import numpy as np
from ..base import BaseEstimator
from .base import SelectorMixin
from ..utils import check_array
from ..utils.sparsefuncs import mean_variance_axis
from ..utils.validation import check_is_fitted

class VarianceThreshold(BaseEstimator, SelectorMixin):
    """Feature selector that removes all low-variance features.

    This feature selection algorithm looks only at the features (X), not the
    desired outputs (y), and can thus be used for unsupervised learning.

    Read more in the :ref:`User Guide <variance_threshold>`.

    Parameters
    ----------
    threshold : float, optional
        Features with a training-set variance lower than this threshold will
        be removed. The default is to keep all features with non-zero variance,
        i.e. remove the features that have the same value in all samples.

    Attributes
    ----------
    variances_ : array, shape (n_features,)
        Variances of individual features.

    Examples
    --------
    The following dataset has integer features, two of which are the same
    in every sample. These are removed with the default setting for threshold::

        >>> X = [[0, 2, 0, 3], [0, 1, 4, 3], [0, 1, 1, 3]]
        >>> selector = VarianceThreshold()
        >>> selector.fit_transform(X)
        array([[2, 0],
               [1, 4],
               [1, 1]])
    """

    def __init__(self, threshold=0.):
        self.threshold = threshold

    def fit(self, X, y=None):
        """Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self
        """
        X = check_array(X, ('csr', 'csc'), dtype=np.float64)

        if hasattr(X, "toarray"):   # sparse matrix
            _, self.variances_ = mean_variance_axis(X, axis=0)
        else:
            self.variances_ = np.var(X, axis=0)

        if np.all(self.variances_ <= self.threshold):
            msg = "No feature in X meets the variance threshold {0:.5f}"
            if X.shape[0] == 1:
                msg += " (X contains only one sample)"
            raise ValueError(msg.format(self.threshold))

        return self

    def _get_support_mask(self):
        check_is_fitted(self, 'variances_')

        return self.variances_ > self.threshold